From 6937c5f0add82491e3b4bc60912d16aa48813c1b Mon Sep 17 00:00:00 2001 From: zhshw Date: Mon, 14 Mar 2022 14:17:00 +0800 Subject: [PATCH 01/19] init xds client, balancer from grpc --- go.mod | 8 +- go.sum | 7 + xds/balancer/balancer.go | 29 + xds/balancer/cdsbalancer/cdsbalancer.go | 547 ++++++++++ xds/balancer/cdsbalancer/cluster_handler.go | 319 ++++++ xds/balancer/cdsbalancer/logging.go | 34 + xds/balancer/clusterimpl/clusterimpl.go | 543 ++++++++++ xds/balancer/clusterimpl/config.go | 64 ++ xds/balancer/clusterimpl/config_test.go | 144 +++ xds/balancer/clusterimpl/logging.go | 34 + xds/balancer/clusterimpl/picker.go | 189 ++++ .../clustermanager/balancerstateaggregator.go | 222 ++++ xds/balancer/clustermanager/clustermanager.go | 149 +++ xds/balancer/clustermanager/config.go | 46 + xds/balancer/clustermanager/picker.go | 70 ++ .../clusterresolver/clusterresolver.go | 379 +++++++ xds/balancer/clusterresolver/config.go | 185 ++++ xds/balancer/clusterresolver/configbuilder.go | 363 +++++++ xds/balancer/clusterresolver/logging.go | 34 + .../clusterresolver/resource_resolver.go | 248 +++++ .../clusterresolver/resource_resolver_dns.go | 114 ++ .../clusterresolver/weightedtarget_config.go | 48 + xds/balancer/loadstore/load_store_wrapper.go | 120 +++ xds/balancer/orca/orca.go | 84 ++ xds/balancer/priority/balancer.go | 253 +++++ xds/balancer/priority/balancer_child.go | 112 ++ xds/balancer/priority/balancer_priority.go | 362 +++++++ xds/balancer/priority/config.go | 67 ++ xds/balancer/priority/config_test.go | 108 ++ xds/balancer/priority/ignore_resolve_now.go | 73 ++ xds/balancer/priority/logging.go | 34 + xds/balancer/priority/utils.go | 31 + xds/balancer/priority/utils_test.go | 62 ++ xds/balancer/ringhash/config.go | 56 + xds/balancer/ringhash/config_test.go | 68 ++ xds/balancer/ringhash/logging.go | 34 + xds/balancer/ringhash/picker.go | 154 +++ xds/balancer/ringhash/ring.go | 163 +++ xds/balancer/ringhash/ring_test.go | 113 ++ xds/balancer/ringhash/ringhash.go | 434 ++++++++ xds/balancer/ringhash/util.go | 40 + xds/client/attributes.go | 60 ++ xds/client/authority.go | 228 ++++ xds/client/bootstrap/bootstrap.go | 476 +++++++++ xds/client/bootstrap/bootstrap_test.go | 992 ++++++++++++++++++ xds/client/bootstrap/logging.go | 28 + xds/client/bootstrap/template.go | 47 + xds/client/bootstrap/template_test.go | 97 ++ xds/client/client.go | 163 +++ xds/client/controller.go | 38 + xds/client/controller/controller.go | 168 +++ xds/client/controller/loadreport.go | 144 +++ xds/client/controller/transport.go | 429 ++++++++ xds/client/controller/version/v2/client.go | 155 +++ .../controller/version/v2/loadreport.go | 153 +++ xds/client/controller/version/v3/client.go | 157 +++ .../controller/version/v3/loadreport.go | 152 +++ xds/client/controller/version/version.go | 123 +++ xds/client/dump.go | 63 ++ xds/client/load/reporter.go | 27 + xds/client/load/store.go | 426 ++++++++ xds/client/load/store_test.go | 446 ++++++++ xds/client/loadreport.go | 47 + xds/client/logging.go | 34 + xds/client/pubsub/dump.go | 87 ++ xds/client/pubsub/interface.go | 39 + xds/client/pubsub/pubsub.go | 182 ++++ xds/client/pubsub/update.go | 318 ++++++ xds/client/pubsub/watch.go | 232 ++++ xds/client/requests_counter.go | 107 ++ xds/client/resource/errors.go | 60 ++ xds/client/resource/filter_chain.go | 872 +++++++++++++++ xds/client/resource/locality_id.go | 82 ++ xds/client/resource/matcher.go | 275 +++++ xds/client/resource/matcher_path.go | 102 ++ xds/client/resource/name.go | 130 +++ xds/client/resource/type.go | 150 +++ xds/client/resource/type_cds.go | 87 ++ xds/client/resource/type_eds.go | 79 ++ xds/client/resource/type_lds.go | 87 ++ xds/client/resource/type_rds.go | 255 +++++ xds/client/resource/unmarshal.go | 178 ++++ xds/client/resource/unmarshal_cds.go | 456 ++++++++ xds/client/resource/unmarshal_eds.go | 130 +++ xds/client/resource/unmarshal_lds.go | 297 ++++++ xds/client/resource/unmarshal_rds.go | 440 ++++++++ xds/client/resource/version/version.go | 63 ++ xds/client/singleton.go | 201 ++++ xds/client/watchers.go | 105 ++ xds/clusterspecifier/cluster_specifier.go | 67 ++ xds/httpfilter/fault/fault.go | 301 ++++++ xds/httpfilter/httpfilter.go | 108 ++ xds/httpfilter/rbac/rbac.go | 220 ++++ xds/httpfilter/router/router.go | 114 ++ xds/internal.go | 88 ++ xds/resolver/logging.go | 34 + xds/resolver/serviceconfig.go | 440 ++++++++ xds/resolver/watch_service.go | 199 ++++ xds/resolver/xds_resolver.go | 315 ++++++ xds/utils/backoff/backoff.go | 73 ++ xds/utils/balancer/stub/stub.go | 104 ++ xds/utils/balancergroup/balancergroup.go | 531 ++++++++++ .../balancergroup/balancerstateaggregator.go | 37 + xds/utils/balancerload/load.go | 46 + xds/utils/buffer/unbounded.go | 85 ++ xds/utils/credentials/xds/handshake_info.go | 318 ++++++ .../credentials/xds/handshake_info_test.go | 304 ++++++ xds/utils/envconfig/envconfig.go | 35 + xds/utils/envconfig/xds.go | 97 ++ xds/utils/grpclog/grpclog.go | 126 +++ xds/utils/grpclog/prefixLogger.go | 81 ++ xds/utils/grpcrand/grpcrand.go | 67 ++ xds/utils/grpcsync/event.go | 61 ++ xds/utils/grpcutil/encode_duration.go | 63 ++ xds/utils/grpcutil/encode_duration_test.go | 51 + xds/utils/grpcutil/grpcutil.go | 20 + xds/utils/grpcutil/metadata.go | 40 + xds/utils/grpcutil/method.go | 84 ++ xds/utils/grpcutil/method_test.go | 69 ++ xds/utils/grpcutil/regex.go | 31 + xds/utils/grpcutil/regex_test.go | 72 ++ xds/utils/hierarchy/hierarchy.go | 109 ++ xds/utils/hierarchy/hierarchy_test.go | 197 ++++ xds/utils/matcher/matcher_header.go | 241 +++++ xds/utils/matcher/matcher_header_test.go | 469 +++++++++ xds/utils/matcher/regex.go | 31 + xds/utils/matcher/regex_test.go | 72 ++ xds/utils/matcher/string_matcher.go | 184 ++++ xds/utils/matcher/string_matcher_test.go | 315 ++++++ xds/utils/metadata/metadata.go | 40 + xds/utils/pretty/pretty.go | 82 ++ xds/utils/rbac/matchers.go | 423 ++++++++ xds/utils/rbac/rbac_engine.go | 230 ++++ xds/utils/resolver/config_selector.go | 167 +++ xds/utils/resolver/passthrough/passthrough.go | 57 + xds/utils/resolver/unix/unix.go | 73 ++ xds/utils/serviceconfig/serviceconfig.go | 180 ++++ xds/utils/serviceconfig/serviceconfig_test.go | 182 ++++ xds/utils/wrr/edf.go | 92 ++ xds/utils/wrr/random.go | 79 ++ xds/utils/wrr/wrr.go | 32 + xds/xds_handshake_cluster.go | 40 + 142 files changed, 23386 insertions(+), 1 deletion(-) create mode 100644 xds/balancer/balancer.go create mode 100644 xds/balancer/cdsbalancer/cdsbalancer.go create mode 100644 xds/balancer/cdsbalancer/cluster_handler.go create mode 100644 xds/balancer/cdsbalancer/logging.go create mode 100644 xds/balancer/clusterimpl/clusterimpl.go create mode 100644 xds/balancer/clusterimpl/config.go create mode 100644 xds/balancer/clusterimpl/config_test.go create mode 100644 xds/balancer/clusterimpl/logging.go create mode 100644 xds/balancer/clusterimpl/picker.go create mode 100644 xds/balancer/clustermanager/balancerstateaggregator.go create mode 100644 xds/balancer/clustermanager/clustermanager.go create mode 100644 xds/balancer/clustermanager/config.go create mode 100644 xds/balancer/clustermanager/picker.go create mode 100644 xds/balancer/clusterresolver/clusterresolver.go create mode 100644 xds/balancer/clusterresolver/config.go create mode 100644 xds/balancer/clusterresolver/configbuilder.go create mode 100644 xds/balancer/clusterresolver/logging.go create mode 100644 xds/balancer/clusterresolver/resource_resolver.go create mode 100644 xds/balancer/clusterresolver/resource_resolver_dns.go create mode 100644 xds/balancer/clusterresolver/weightedtarget_config.go create mode 100644 xds/balancer/loadstore/load_store_wrapper.go create mode 100644 xds/balancer/orca/orca.go create mode 100644 xds/balancer/priority/balancer.go create mode 100644 xds/balancer/priority/balancer_child.go create mode 100644 xds/balancer/priority/balancer_priority.go create mode 100644 xds/balancer/priority/config.go create mode 100644 xds/balancer/priority/config_test.go create mode 100644 xds/balancer/priority/ignore_resolve_now.go create mode 100644 xds/balancer/priority/logging.go create mode 100644 xds/balancer/priority/utils.go create mode 100644 xds/balancer/priority/utils_test.go create mode 100644 xds/balancer/ringhash/config.go create mode 100644 xds/balancer/ringhash/config_test.go create mode 100644 xds/balancer/ringhash/logging.go create mode 100644 xds/balancer/ringhash/picker.go create mode 100644 xds/balancer/ringhash/ring.go create mode 100644 xds/balancer/ringhash/ring_test.go create mode 100644 xds/balancer/ringhash/ringhash.go create mode 100644 xds/balancer/ringhash/util.go create mode 100644 xds/client/attributes.go create mode 100644 xds/client/authority.go create mode 100644 xds/client/bootstrap/bootstrap.go create mode 100644 xds/client/bootstrap/bootstrap_test.go create mode 100644 xds/client/bootstrap/logging.go create mode 100644 xds/client/bootstrap/template.go create mode 100644 xds/client/bootstrap/template_test.go create mode 100644 xds/client/client.go create mode 100644 xds/client/controller.go create mode 100644 xds/client/controller/controller.go create mode 100644 xds/client/controller/loadreport.go create mode 100644 xds/client/controller/transport.go create mode 100644 xds/client/controller/version/v2/client.go create mode 100644 xds/client/controller/version/v2/loadreport.go create mode 100644 xds/client/controller/version/v3/client.go create mode 100644 xds/client/controller/version/v3/loadreport.go create mode 100644 xds/client/controller/version/version.go create mode 100644 xds/client/dump.go create mode 100644 xds/client/load/reporter.go create mode 100644 xds/client/load/store.go create mode 100644 xds/client/load/store_test.go create mode 100644 xds/client/loadreport.go create mode 100644 xds/client/logging.go create mode 100644 xds/client/pubsub/dump.go create mode 100644 xds/client/pubsub/interface.go create mode 100644 xds/client/pubsub/pubsub.go create mode 100644 xds/client/pubsub/update.go create mode 100644 xds/client/pubsub/watch.go create mode 100644 xds/client/requests_counter.go create mode 100644 xds/client/resource/errors.go create mode 100644 xds/client/resource/filter_chain.go create mode 100644 xds/client/resource/locality_id.go create mode 100644 xds/client/resource/matcher.go create mode 100644 xds/client/resource/matcher_path.go create mode 100644 xds/client/resource/name.go create mode 100644 xds/client/resource/type.go create mode 100644 xds/client/resource/type_cds.go create mode 100644 xds/client/resource/type_eds.go create mode 100644 xds/client/resource/type_lds.go create mode 100644 xds/client/resource/type_rds.go create mode 100644 xds/client/resource/unmarshal.go create mode 100644 xds/client/resource/unmarshal_cds.go create mode 100644 xds/client/resource/unmarshal_eds.go create mode 100644 xds/client/resource/unmarshal_lds.go create mode 100644 xds/client/resource/unmarshal_rds.go create mode 100644 xds/client/resource/version/version.go create mode 100644 xds/client/singleton.go create mode 100644 xds/client/watchers.go create mode 100644 xds/clusterspecifier/cluster_specifier.go create mode 100644 xds/httpfilter/fault/fault.go create mode 100644 xds/httpfilter/httpfilter.go create mode 100644 xds/httpfilter/rbac/rbac.go create mode 100644 xds/httpfilter/router/router.go create mode 100644 xds/internal.go create mode 100644 xds/resolver/logging.go create mode 100644 xds/resolver/serviceconfig.go create mode 100644 xds/resolver/watch_service.go create mode 100644 xds/resolver/xds_resolver.go create mode 100644 xds/utils/backoff/backoff.go create mode 100644 xds/utils/balancer/stub/stub.go create mode 100644 xds/utils/balancergroup/balancergroup.go create mode 100644 xds/utils/balancergroup/balancerstateaggregator.go create mode 100644 xds/utils/balancerload/load.go create mode 100644 xds/utils/buffer/unbounded.go create mode 100644 xds/utils/credentials/xds/handshake_info.go create mode 100644 xds/utils/credentials/xds/handshake_info_test.go create mode 100644 xds/utils/envconfig/envconfig.go create mode 100644 xds/utils/envconfig/xds.go create mode 100644 xds/utils/grpclog/grpclog.go create mode 100644 xds/utils/grpclog/prefixLogger.go create mode 100644 xds/utils/grpcrand/grpcrand.go create mode 100644 xds/utils/grpcsync/event.go create mode 100644 xds/utils/grpcutil/encode_duration.go create mode 100644 xds/utils/grpcutil/encode_duration_test.go create mode 100644 xds/utils/grpcutil/grpcutil.go create mode 100644 xds/utils/grpcutil/metadata.go create mode 100644 xds/utils/grpcutil/method.go create mode 100644 xds/utils/grpcutil/method_test.go create mode 100644 xds/utils/grpcutil/regex.go create mode 100644 xds/utils/grpcutil/regex_test.go create mode 100644 xds/utils/hierarchy/hierarchy.go create mode 100644 xds/utils/hierarchy/hierarchy_test.go create mode 100644 xds/utils/matcher/matcher_header.go create mode 100644 xds/utils/matcher/matcher_header_test.go create mode 100644 xds/utils/matcher/regex.go create mode 100644 xds/utils/matcher/regex_test.go create mode 100644 xds/utils/matcher/string_matcher.go create mode 100644 xds/utils/matcher/string_matcher_test.go create mode 100644 xds/utils/metadata/metadata.go create mode 100644 xds/utils/pretty/pretty.go create mode 100644 xds/utils/rbac/matchers.go create mode 100644 xds/utils/rbac/rbac_engine.go create mode 100644 xds/utils/resolver/config_selector.go create mode 100644 xds/utils/resolver/passthrough/passthrough.go create mode 100644 xds/utils/resolver/unix/unix.go create mode 100644 xds/utils/serviceconfig/serviceconfig.go create mode 100644 xds/utils/serviceconfig/serviceconfig_test.go create mode 100644 xds/utils/wrr/edf.go create mode 100644 xds/utils/wrr/random.go create mode 100644 xds/utils/wrr/wrr.go create mode 100644 xds/xds_handshake_cluster.go diff --git a/go.mod b/go.mod index 01826d98d8..34d39834cc 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,16 @@ require ( github.com/alibaba/sentinel-golang v1.0.4 github.com/apache/dubbo-getty v1.4.7 github.com/apache/dubbo-go-hessian2 v1.11.0 + github.com/cespare/xxhash/v2 v2.1.2 + github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 + github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 github.com/creasty/defaults v1.5.2 github.com/dubbogo/go-zookeeper v1.0.4-0.20211212162352-f9d2183d89d5 github.com/dubbogo/gost v1.11.25 github.com/dubbogo/grpc-go v1.42.8 github.com/dubbogo/triple v1.1.8-rc2 github.com/emicklei/go-restful/v3 v3.7.3 + github.com/envoyproxy/go-control-plane v0.10.0 github.com/fsnotify/fsnotify v1.5.1 github.com/ghodss/yaml v1.0.0 github.com/go-co-op/gocron v1.9.0 @@ -23,6 +27,7 @@ require ( github.com/go-resty/resty/v2 v2.7.0 github.com/golang/mock v1.4.4 github.com/golang/protobuf v1.5.2 + github.com/google/go-cmp v0.5.6 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 github.com/hashicorp/vault/sdk v0.3.0 github.com/jinzhu/copier v0.3.5 @@ -44,7 +49,8 @@ require ( go.etcd.io/etcd/server/v3 v3.5.2 go.uber.org/atomic v1.9.0 go.uber.org/zap v1.21.0 - google.golang.org/grpc v1.44.0 + google.golang.org/genproto v0.0.0-20211104193956-4c6863e31247 + google.golang.org/grpc v1.45.0 google.golang.org/protobuf v1.27.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.22.4 diff --git a/go.sum b/go.sum index 29c7c8f9d7..481591584b 100644 --- a/go.sum +++ b/go.sum @@ -126,6 +126,7 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 h1:uH66TXeswKn5PW5zdZ39xEwfS9an067BirqA+P4QaLI= @@ -144,11 +145,13 @@ github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 h1:hzAQntlaYRkVSFEfj9OTWlVV1H155FMD8BTKktLv0QI= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1 h1:zH8ljVhhq7yC0MIeUL/IviMtY8hx2mK8cN9wEYb8ggw= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5 h1:xD/lrqdvwsc+O2bjSSi3YqY73Ke3LAiSCx49aCesA0E= @@ -225,7 +228,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.0 h1:WVt4HEPbdRbRD/PKKPbPnIVavO6gk/h673jWyIJ016k= github.com/envoyproxy/go-control-plane v0.10.0/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= +github.com/envoyproxy/protoc-gen-validate v0.1.0 h1:EQciDnbrYxy13PgWoY8AqoxGiPrpgBZ1R8UNe3ddc+A= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -1303,6 +1308,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go new file mode 100644 index 0000000000..5ac911301d --- /dev/null +++ b/xds/balancer/balancer.go @@ -0,0 +1,29 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package balancer installs all the xds balancers. +package balancer + +import ( + _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/cdsbalancer" // Register the CDS balancer + _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" // Register the xds_cluster_impl balancer + _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" // Register the xds_cluster_manager balancer + _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" // Register the xds_cluster_resolver balancer + _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" // Register the priority balancer + _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer +) diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go new file mode 100644 index 0000000000..9ab069e1de --- /dev/null +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -0,0 +1,547 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cdsbalancer implements a balancer to handle CDS responses. +package cdsbalancer + +import ( + "encoding/json" + "errors" + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + cdsName = "cds_experimental" +) + +var ( + errBalancerClosed = errors.New("cdsBalancer is closed") + + // newChildBalancer is a helper function to build a new cluster_resolver + // balancer and will be overridden in unittests. + newChildBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions) (balancer.Balancer, error) { + builder := balancer.Get(clusterresolver.Name) + if builder == nil { + return nil, fmt.Errorf("xds: no balancer builder with name %v", clusterresolver.Name) + } + // We directly pass the parent clientConn to the underlying + // cluster_resolver balancer because the cdsBalancer does not deal with + // subConns. + return builder.Build(cc, opts), nil + } + buildProvider = buildProviderFunc +) + +func init() { + balancer.Register(bb{}) +} + +// bb implements the balancer.Builder interface to help build a cdsBalancer. +// It also implements the balancer.ConfigParser interface to help parse the +// JSON service config, to be passed to the cdsBalancer. +type bb struct{} + +// Build creates a new CDS balancer with the ClientConn. +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &cdsBalancer{ + bOpts: opts, + updateCh: buffer.NewUnbounded(), + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + xdsHI: xdsinternal.NewHandshakeInfo(nil, nil), + } + b.logger = prefixLogger((b)) + b.logger.Infof("Created") + var creds credentials.TransportCredentials + switch { + case opts.DialCreds != nil: + creds = opts.DialCreds + case opts.CredsBundle != nil: + creds = opts.CredsBundle.TransportCredentials() + } + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + b.xdsCredsInUse = true + } + b.logger.Infof("xDS credentials in use: %v", b.xdsCredsInUse) + b.clusterHandler = newClusterHandler(b) + b.ccw = &ccWrapper{ + ClientConn: cc, + xdsHI: b.xdsHI, + } + go b.run() + return b +} + +// Name returns the name of balancers built by this builder. +func (bb) Name() string { + return cdsName +} + +// lbConfig represents the loadBalancingConfig section of the service config +// for the cdsBalancer. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + ClusterName string `json:"Cluster"` +} + +// ParseConfig parses the JSON load balancer config provided into an +// internal form or returns an error if the config is invalid. +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg lbConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, fmt.Errorf("xds: unable to unmarshal lbconfig: %s, error: %v", string(c), err) + } + return &cfg, nil +} + +// ccUpdate wraps a clientConn update received from gRPC (pushed from the +// xdsResolver). A valid clusterName causes the cdsBalancer to register a CDS +// watcher with the xdsClient, while a non-nil error causes it to cancel the +// existing watch and propagate the error to the underlying cluster_resolver +// balancer. +type ccUpdate struct { + clusterName string + err error +} + +// scUpdate wraps a subConn update received from gRPC. This is directly passed +// on to the cluster_resolver balancer. +type scUpdate struct { + subConn balancer.SubConn + state balancer.SubConnState +} + +type exitIdle struct{} + +// cdsBalancer implements a CDS based LB policy. It instantiates a +// cluster_resolver balancer to further resolve the serviceName received from +// CDS, into localities and endpoints. Implements the balancer.Balancer +// interface which is exposed to gRPC and implements the balancer.ClientConn +// interface which is exposed to the cluster_resolver balancer. +type cdsBalancer struct { + ccw *ccWrapper // ClientConn interface passed to child LB. + bOpts balancer.BuildOptions // BuildOptions passed to child LB. + updateCh *buffer.Unbounded // Channel for gRPC and xdsClient updates. + xdsClient client.XDSClient // xDS client to watch Cluster resource. + clusterHandler *clusterHandler // To watch the clusters. + childLB balancer.Balancer + logger *grpclog.PrefixLogger + closed *grpcsync.Event + done *grpcsync.Event + + // The certificate providers are cached here to that they can be closed when + // a new provider is to be created. + cachedRoot certprovider.Provider + cachedIdentity certprovider.Provider + xdsHI *xdsinternal.HandshakeInfo + xdsCredsInUse bool +} + +// handleClientConnUpdate handles a ClientConnUpdate received from gRPC. Good +// updates lead to registration of a CDS watch. Updates with error lead to +// cancellation of existing watch and propagation of the same error to the +// cluster_resolver balancer. +func (b *cdsBalancer) handleClientConnUpdate(update *ccUpdate) { + // We first handle errors, if any, and then proceed with handling the + // update, only if the status quo has changed. + if err := update.err; err != nil { + b.handleErrorFromUpdate(err, true) + return + } + b.clusterHandler.updateRootCluster(update.clusterName) +} + +// handleSecurityConfig processes the security configuration received from the +// management server, creates appropriate certificate provider plugins, and +// updates the HandhakeInfo which is added as an address attribute in +// NewSubConn() calls. +func (b *cdsBalancer) handleSecurityConfig(config *resource.SecurityConfig) error { + // If xdsCredentials are not in use, i.e, the user did not want to get + // security configuration from an xDS server, we should not be acting on the + // received security config here. Doing so poses a security threat. + if !b.xdsCredsInUse { + return nil + } + + // Security config being nil is a valid case where the management server has + // not sent any security configuration. The xdsCredentials implementation + // handles this by delegating to its fallback credentials. + if config == nil { + // We need to explicitly set the fields to nil here since this might be + // a case of switching from a good security configuration to an empty + // one where fallback credentials are to be used. + b.xdsHI.SetRootCertProvider(nil) + b.xdsHI.SetIdentityCertProvider(nil) + b.xdsHI.SetSANMatchers(nil) + return nil + } + + bc := b.xdsClient.BootstrapConfig() + if bc == nil || bc.CertProviderConfigs == nil { + // Bootstrap did not find any certificate provider configs, but the user + // has specified xdsCredentials and the management server has sent down + // security configuration. + return errors.New("xds: certificate_providers config missing in bootstrap file") + } + cpc := bc.CertProviderConfigs + + // A root provider is required whether we are using TLS or mTLS. + rootProvider, err := buildProvider(cpc, config.RootInstanceName, config.RootCertName, false, true) + if err != nil { + return err + } + + // The identity provider is only present when using mTLS. + var identityProvider certprovider.Provider + if name, cert := config.IdentityInstanceName, config.IdentityCertName; name != "" { + var err error + identityProvider, err = buildProvider(cpc, name, cert, true, false) + if err != nil { + return err + } + } + + // Close the old providers and cache the new ones. + if b.cachedRoot != nil { + b.cachedRoot.Close() + } + if b.cachedIdentity != nil { + b.cachedIdentity.Close() + } + b.cachedRoot = rootProvider + b.cachedIdentity = identityProvider + + // We set all fields here, even if some of them are nil, since they + // could have been non-nil earlier. + b.xdsHI.SetRootCertProvider(rootProvider) + b.xdsHI.SetIdentityCertProvider(identityProvider) + b.xdsHI.SetSANMatchers(config.SubjectAltNameMatchers) + return nil +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("xds: failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} + +// handleWatchUpdate handles a watch update from the xDS Client. Good updates +// lead to clientConn updates being invoked on the underlying cluster_resolver balancer. +func (b *cdsBalancer) handleWatchUpdate(update clusterHandlerUpdate) { + if err := update.err; err != nil { + b.logger.Warningf("Watch error from xds-client %p: %v", b.xdsClient, err) + b.handleErrorFromUpdate(err, false) + return + } + + b.logger.Infof("Watch update from xds-client %p, content: %+v, security config: %v", b.xdsClient, pretty.ToJSON(update.updates), pretty.ToJSON(update.securityCfg)) + + // Process the security config from the received update before building the + // child policy or forwarding the update to it. We do this because the child + // policy may try to create a new subConn inline. Processing the security + // configuration here and setting up the handshakeInfo will make sure that + // such attempts are handled properly. + if err := b.handleSecurityConfig(update.securityCfg); err != nil { + // If the security config is invalid, for example, if the provider + // instance is not found in the bootstrap config, we need to put the + // channel in transient failure. + b.logger.Warningf("Invalid security config update from xds-client %p: %v", b.xdsClient, err) + b.handleErrorFromUpdate(err, false) + return + } + + // The first good update from the watch API leads to the instantiation of an + // cluster_resolver balancer. Further updates/errors are propagated to the existing + // cluster_resolver balancer. + if b.childLB == nil { + childLB, err := newChildBalancer(b.ccw, b.bOpts) + if err != nil { + b.logger.Errorf("Failed to create child policy of type %s, %v", clusterresolver.Name, err) + return + } + b.childLB = childLB + b.logger.Infof("Created child policy %p of type %s", b.childLB, clusterresolver.Name) + } + + dms := make([]clusterresolver.DiscoveryMechanism, len(update.updates)) + for i, cu := range update.updates { + switch cu.ClusterType { + case resource.ClusterTypeEDS: + dms[i] = clusterresolver.DiscoveryMechanism{ + Type: clusterresolver.DiscoveryMechanismTypeEDS, + Cluster: cu.ClusterName, + EDSServiceName: cu.EDSServiceName, + MaxConcurrentRequests: cu.MaxRequests, + } + if cu.EnableLRS { + // An empty string here indicates that the cluster_resolver balancer should use the + // same xDS server for load reporting as it does for EDS + // requests/responses. + dms[i].LoadReportingServerName = new(string) + + } + case resource.ClusterTypeLogicalDNS: + dms[i] = clusterresolver.DiscoveryMechanism{ + Type: clusterresolver.DiscoveryMechanismTypeLogicalDNS, + DNSHostname: cu.DNSHostName, + } + default: + b.logger.Infof("unexpected cluster type %v when handling update from cluster handler", cu.ClusterType) + } + } + lbCfg := &clusterresolver.LBConfig{ + DiscoveryMechanisms: dms, + } + + // lbPolicy is set only when the policy is ringhash. The default (when it's + // not set) is roundrobin. And similarly, we only need to set XDSLBPolicy + // for ringhash (it also defaults to roundrobin). + if lbp := update.lbPolicy; lbp != nil { + lbCfg.XDSLBPolicy = &internalserviceconfig.BalancerConfig{ + Name: ringhash.Name, + Config: &ringhash.LBConfig{ + MinRingSize: lbp.MinimumRingSize, + MaxRingSize: lbp.MaximumRingSize, + }, + } + } + + ccState := balancer.ClientConnState{ + ResolverState: client.SetClient(resolver.State{}, b.xdsClient), + BalancerConfig: lbCfg, + } + if err := b.childLB.UpdateClientConnState(ccState); err != nil { + b.logger.Errorf("xds: cluster_resolver balancer.UpdateClientConnState(%+v) returned error: %v", ccState, err) + } +} + +// run is a long-running goroutine which handles all updates from gRPC. All +// methods which are invoked directly by gRPC or xdsClient simply push an +// update onto a channel which is read and acted upon right here. +func (b *cdsBalancer) run() { + for { + select { + case u := <-b.updateCh.Get(): + b.updateCh.Load() + switch update := u.(type) { + case *ccUpdate: + b.handleClientConnUpdate(update) + case *scUpdate: + // SubConn updates are passthrough and are simply handed over to + // the underlying cluster_resolver balancer. + if b.childLB == nil { + b.logger.Errorf("xds: received scUpdate {%+v} with no cluster_resolver balancer", update) + break + } + b.childLB.UpdateSubConnState(update.subConn, update.state) + case exitIdle: + if b.childLB == nil { + b.logger.Errorf("xds: received ExitIdle with no child balancer") + break + } + // This implementation assumes the child balancer supports + // ExitIdle (but still checks for the interface's existence to + // avoid a panic if not). If the child does not, no subconns + // will be connected. + if ei, ok := b.childLB.(balancer.ExitIdler); ok { + ei.ExitIdle() + } + } + case u := <-b.clusterHandler.updateChannel: + b.handleWatchUpdate(u) + case <-b.closed.Done(): + b.clusterHandler.close() + if b.childLB != nil { + b.childLB.Close() + b.childLB = nil + } + if b.cachedRoot != nil { + b.cachedRoot.Close() + } + if b.cachedIdentity != nil { + b.cachedIdentity.Close() + } + b.logger.Infof("Shutdown") + b.done.Fire() + return + } + } +} + +// handleErrorFromUpdate handles both the error from parent ClientConn (from +// resolver) and the error from xds client (from the watcher). fromParent is +// true if error is from parent ClientConn. +// +// If the error is connection error, it's passed down to the child policy. +// Nothing needs to be done in CDS (e.g. it doesn't go into fallback). +// +// If the error is resource-not-found: +// - If it's from resolver, it means LDS resources were removed. The CDS watch +// should be canceled. +// - If it's from xds client, it means CDS resource were removed. The CDS +// watcher should keep watching. +// +// In both cases, the error will be forwarded to the child balancer. And if +// error is resource-not-found, the child balancer will stop watching EDS. +func (b *cdsBalancer) handleErrorFromUpdate(err error, fromParent bool) { + // This is not necessary today, because xds client never sends connection + // errors. + if fromParent && resource.ErrType(err) == resource.ErrorTypeResourceNotFound { + b.clusterHandler.close() + } + if b.childLB != nil { + if resource.ErrType(err) != resource.ErrorTypeConnection { + // Connection errors will be sent to the child balancers directly. + // There's no need to forward them. + b.childLB.ResolverError(err) + } + } else { + // If child balancer was never created, fail the RPCs with + // errors. + b.ccw.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) + } +} + +// UpdateClientConnState receives the serviceConfig (which contains the +// clusterName to watch for in CDS) and the xdsClient object from the +// xdsResolver. +func (b *cdsBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if b.closed.HasFired() { + b.logger.Warningf("xds: received ClientConnState {%+v} after cdsBalancer was closed", state) + return errBalancerClosed + } + + if b.xdsClient == nil { + c := client.FromResolverState(state.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(state.BalancerConfig)) + // The errors checked here should ideally never happen because the + // ServiceConfig in this case is prepared by the xdsResolver and is not + // something that is received on the wire. + lbCfg, ok := state.BalancerConfig.(*lbConfig) + if !ok { + b.logger.Warningf("xds: unexpected LoadBalancingConfig type: %T", state.BalancerConfig) + return balancer.ErrBadResolverState + } + if lbCfg.ClusterName == "" { + b.logger.Warningf("xds: no clusterName found in LoadBalancingConfig: %+v", lbCfg) + return balancer.ErrBadResolverState + } + b.updateCh.Put(&ccUpdate{clusterName: lbCfg.ClusterName}) + return nil +} + +// ResolverError handles errors reported by the xdsResolver. +func (b *cdsBalancer) ResolverError(err error) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received resolver error {%v} after cdsBalancer was closed", err) + return + } + b.updateCh.Put(&ccUpdate{err: err}) +} + +// UpdateSubConnState handles subConn updates from gRPC. +func (b *cdsBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received subConn update {%v, %v} after cdsBalancer was closed", sc, state) + return + } + b.updateCh.Put(&scUpdate{subConn: sc, state: state}) +} + +// Close cancels the CDS watch, closes the child policy and closes the +// cdsBalancer. +func (b *cdsBalancer) Close() { + b.closed.Fire() + <-b.done.Done() +} + +func (b *cdsBalancer) ExitIdle() { + b.updateCh.Put(exitIdle{}) +} + +// ccWrapper wraps the balancer.ClientConn passed to the CDS balancer at +// creation and intercepts the NewSubConn() and UpdateAddresses() call from the +// child policy to add security configuration required by xDS credentials. +// +// Other methods of the balancer.ClientConn interface are not overridden and +// hence get the original implementation. +type ccWrapper struct { + balancer.ClientConn + + // The certificate providers in this HandshakeInfo are updated based on the + // received security configuration in the Cluster resource. + xdsHI *xdsinternal.HandshakeInfo +} + +// NewSubConn intercepts NewSubConn() calls from the child policy and adds an +// address attribute which provides all information required by the xdsCreds +// handshaker to perform the TLS handshake. +func (ccw *ccWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + newAddrs := make([]resolver.Address, len(addrs)) + for i, addr := range addrs { + newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI) + } + return ccw.ClientConn.NewSubConn(newAddrs, opts) +} + +func (ccw *ccWrapper) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + newAddrs := make([]resolver.Address, len(addrs)) + for i, addr := range addrs { + newAddrs[i] = xdsinternal.SetHandshakeInfo(addr, ccw.xdsHI) + } + ccw.ClientConn.UpdateAddresses(sc, newAddrs) +} diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go new file mode 100644 index 0000000000..137825309b --- /dev/null +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -0,0 +1,319 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cdsbalancer + +import ( + "errors" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +var errNotReceivedUpdate = errors.New("tried to construct a cluster update on a cluster that has not received an update") + +// clusterHandlerUpdate wraps the information received from the registered CDS +// watcher. A non-nil error is propagated to the underlying cluster_resolver +// balancer. A valid update results in creating a new cluster_resolver balancer +// (if one doesn't already exist) and pushing the update to it. +type clusterHandlerUpdate struct { + // securityCfg is the Security Config from the top (root) cluster. + securityCfg *resource.SecurityConfig + // lbPolicy is the lb policy from the top (root) cluster. + // + // Currently, we only support roundrobin or ringhash, and since roundrobin + // does need configs, this is only set to the ringhash config, if the policy + // is ringhash. In the future, if we support more policies, we can make this + // an interface, and set it to config of the other policies. + lbPolicy *resource.ClusterLBPolicyRingHash + + // updates is a list of ClusterUpdates from all the leaf clusters. + updates []resource.ClusterUpdate + err error +} + +// clusterHandler will be given a name representing a cluster. It will then +// update the CDS policy constantly with a list of Clusters to pass down to +// XdsClusterResolverLoadBalancingPolicyConfig in a stream like fashion. +type clusterHandler struct { + parent *cdsBalancer + + // A mutex to protect entire tree of clusters. + clusterMutex sync.Mutex + root *clusterNode + rootClusterName string + + // A way to ping CDS Balancer about any updates or errors to a Node in the + // tree. This will either get called from this handler constructing an + // update or from a child with an error. Capacity of one as the only update + // CDS Balancer cares about is the most recent update. + updateChannel chan clusterHandlerUpdate +} + +func newClusterHandler(parent *cdsBalancer) *clusterHandler { + return &clusterHandler{ + parent: parent, + updateChannel: make(chan clusterHandlerUpdate, 1), + } +} + +func (ch *clusterHandler) updateRootCluster(rootClusterName string) { + ch.clusterMutex.Lock() + defer ch.clusterMutex.Unlock() + if ch.root == nil { + // Construct a root node on first update. + ch.root = createClusterNode(rootClusterName, ch.parent.xdsClient, ch) + ch.rootClusterName = rootClusterName + return + } + // Check if root cluster was changed. If it was, delete old one and start + // new one, if not do nothing. + if rootClusterName != ch.rootClusterName { + ch.root.delete() + ch.root = createClusterNode(rootClusterName, ch.parent.xdsClient, ch) + ch.rootClusterName = rootClusterName + } +} + +// This function tries to construct a cluster update to send to CDS. +func (ch *clusterHandler) constructClusterUpdate() { + if ch.root == nil { + // If root is nil, this handler is closed, ignore the update. + return + } + clusterUpdate, err := ch.root.constructClusterUpdate() + if err != nil { + // If there was an error received no op, as this simply means one of the + // children hasn't received an update yet. + return + } + // For a ClusterUpdate, the only update CDS cares about is the most + // recent one, so opportunistically drain the update channel before + // sending the new update. + select { + case <-ch.updateChannel: + default: + } + ch.updateChannel <- clusterHandlerUpdate{ + securityCfg: ch.root.clusterUpdate.SecurityCfg, + lbPolicy: ch.root.clusterUpdate.LBPolicy, + updates: clusterUpdate, + } +} + +// close() is meant to be called by CDS when the CDS balancer is closed, and it +// cancels the watches for every cluster in the cluster tree. +func (ch *clusterHandler) close() { + ch.clusterMutex.Lock() + defer ch.clusterMutex.Unlock() + if ch.root == nil { + return + } + ch.root.delete() + ch.root = nil + ch.rootClusterName = "" +} + +// This logically represents a cluster. This handles all the logic for starting +// and stopping a cluster watch, handling any updates, and constructing a list +// recursively for the ClusterHandler. +type clusterNode struct { + // A way to cancel the watch for the cluster. + cancelFunc func() + + // A list of children, as the Node can be an aggregate Cluster. + children []*clusterNode + + // A ClusterUpdate in order to build a list of cluster updates for CDS to + // send down to child XdsClusterResolverLoadBalancingPolicy. + clusterUpdate resource.ClusterUpdate + + // This boolean determines whether this Node has received an update or not. + // This isn't the best practice, but this will protect a list of Cluster + // Updates from being constructed if a cluster in the tree has not received + // an update yet. + receivedUpdate bool + + clusterHandler *clusterHandler +} + +// CreateClusterNode creates a cluster node from a given clusterName. This will +// also start the watch for that cluster. +func createClusterNode(clusterName string, xdsClient client.XDSClient, topLevelHandler *clusterHandler) *clusterNode { + c := &clusterNode{ + clusterHandler: topLevelHandler, + } + // Communicate with the xds client here. + topLevelHandler.parent.logger.Infof("CDS watch started on %v", clusterName) + cancel := xdsClient.WatchCluster(clusterName, c.handleResp) + c.cancelFunc = func() { + topLevelHandler.parent.logger.Infof("CDS watch canceled on %v", clusterName) + cancel() + } + return c +} + +// This function cancels the cluster watch on the cluster and all of it's +// children. +func (c *clusterNode) delete() { + c.cancelFunc() + for _, child := range c.children { + child.delete() + } +} + +// Construct cluster update (potentially a list of ClusterUpdates) for a node. +func (c *clusterNode) constructClusterUpdate() ([]resource.ClusterUpdate, error) { + // If the cluster has not yet received an update, the cluster update is not + // yet ready. + if !c.receivedUpdate { + return nil, errNotReceivedUpdate + } + + // Base case - LogicalDNS or EDS. Both of these cluster types will be tied + // to a single ClusterUpdate. + if c.clusterUpdate.ClusterType != resource.ClusterTypeAggregate { + return []resource.ClusterUpdate{c.clusterUpdate}, nil + } + + // If an aggregate construct a list by recursively calling down to all of + // it's children. + var childrenUpdates []resource.ClusterUpdate + for _, child := range c.children { + childUpdateList, err := child.constructClusterUpdate() + if err != nil { + return nil, err + } + childrenUpdates = append(childrenUpdates, childUpdateList...) + } + return childrenUpdates, nil +} + +// handleResp handles a xds response for a particular cluster. This function +// also handles any logic with regards to any child state that may have changed. +// At the end of the handleResp(), the clusterUpdate will be pinged in certain +// situations to try and construct an update to send back to CDS. +func (c *clusterNode) handleResp(clusterUpdate resource.ClusterUpdate, err error) { + c.clusterHandler.clusterMutex.Lock() + defer c.clusterHandler.clusterMutex.Unlock() + if err != nil { // Write this error for run() to pick up in CDS LB policy. + // For a ClusterUpdate, the only update CDS cares about is the most + // recent one, so opportunistically drain the update channel before + // sending the new update. + select { + case <-c.clusterHandler.updateChannel: + default: + } + c.clusterHandler.updateChannel <- clusterHandlerUpdate{err: err} + return + } + + c.receivedUpdate = true + c.clusterUpdate = clusterUpdate + + // If the cluster was a leaf node, if the cluster update received had change + // in the cluster update then the overall cluster update would change and + // there is a possibility for the overall update to build so ping cluster + // handler to return. Also, if there was any children from previously, + // delete the children, as the cluster type is no longer an aggregate + // cluster. + if clusterUpdate.ClusterType != resource.ClusterTypeAggregate { + for _, child := range c.children { + child.delete() + } + c.children = nil + // This is an update in the one leaf node, should try to send an update + // to the parent CDS balancer. + // + // Note that this update might be a duplicate from the previous one. + // Because the update contains not only the cluster name to watch, but + // also the extra fields (e.g. security config). There's no good way to + // compare all the fields. + c.clusterHandler.constructClusterUpdate() + return + } + + // Aggregate cluster handling. + newChildren := make(map[string]bool) + for _, childName := range clusterUpdate.PrioritizedClusterNames { + newChildren[childName] = true + } + + // These booleans help determine whether this callback will ping the overall + // clusterHandler to try and construct an update to send back to CDS. This + // will be determined by whether there would be a change in the overall + // clusterUpdate for the whole tree (ex. change in clusterUpdate for current + // cluster or a deleted child) and also if there's even a possibility for + // the update to build (ex. if a child is created and a watch is started, + // that child hasn't received an update yet due to the mutex lock on this + // callback). + var createdChild, deletedChild bool + + // This map will represent the current children of the cluster. It will be + // first added to in order to represent the new children. It will then have + // any children deleted that are no longer present. Then, from the cluster + // update received, will be used to construct the new child list. + mapCurrentChildren := make(map[string]*clusterNode) + for _, child := range c.children { + mapCurrentChildren[child.clusterUpdate.ClusterName] = child + } + + // Add and construct any new child nodes. + for child := range newChildren { + if _, inChildrenAlready := mapCurrentChildren[child]; !inChildrenAlready { + createdChild = true + mapCurrentChildren[child] = createClusterNode(child, c.clusterHandler.parent.xdsClient, c.clusterHandler) + } + } + + // Delete any child nodes no longer in the aggregate cluster's children. + for child := range mapCurrentChildren { + if _, stillAChild := newChildren[child]; !stillAChild { + deletedChild = true + mapCurrentChildren[child].delete() + delete(mapCurrentChildren, child) + } + } + + // The order of the children list matters, so use the clusterUpdate from + // xdsclient as the ordering, and use that logical ordering for the new + // children list. This will be a mixture of child nodes which are all + // already constructed in the mapCurrentChildrenMap. + var children = make([]*clusterNode, 0, len(clusterUpdate.PrioritizedClusterNames)) + + for _, orderedChild := range clusterUpdate.PrioritizedClusterNames { + // The cluster's already have watches started for them in xds client, so + // you can use these pointers to construct the new children list, you + // just have to put them in the correct order using the original cluster + // update. + currentChild := mapCurrentChildren[orderedChild] + children = append(children, currentChild) + } + + c.children = children + + // If the cluster is an aggregate cluster, if this callback created any new + // child cluster nodes, then there's no possibility for a full cluster + // update to successfully build, as those created children will not have + // received an update yet. However, if there was simply a child deleted, + // then there is a possibility that it will have a full cluster update to + // build and also will have a changed overall cluster update from the + // deleted child. + if deletedChild && !createdChild { + c.clusterHandler.constructClusterUpdate() + } +} diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go new file mode 100644 index 0000000000..e0d21c0c97 --- /dev/null +++ b/xds/balancer/cdsbalancer/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package cdsbalancer + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[cds-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *cdsBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go new file mode 100644 index 0000000000..0be70ec12f --- /dev/null +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -0,0 +1,543 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterimpl implements the xds_cluster_impl balancing policy. It +// handles the cluster features (e.g. circuit_breaking, RPC dropping). +// +// Note that it doesn't handle name resolution, which is done by policy +// xds_cluster_resolver. +package clusterimpl + +import ( + internal "dubbo.apache.org/dubbo-go/v3/xds" + "encoding/json" + "fmt" + "sync" + "sync/atomic" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/loadstore" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const ( + // Name is the name of the cluster_impl balancer. + Name = "xds_cluster_impl_experimental" + defaultRequestCountMax = 1024 +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &clusterImplBalancer{ + ClientConn: cc, + bOpts: bOpts, + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + loadWrapper: loadstore.NewWrapper(), + scWrappers: make(map[balancer.SubConn]*scWrapper), + pickerUpdateCh: buffer.NewUnbounded(), + requestCountMax: defaultRequestCountMax, + } + b.logger = prefixLogger(b) + go b.run() + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type clusterImplBalancer struct { + balancer.ClientConn + + // mu guarantees mutual exclusion between Close() and handling of picker + // update to the parent ClientConn in run(). It's to make sure that the + // run() goroutine doesn't send picker update to parent after the balancer + // is closed. + // + // It's only used by the run() goroutine, but not the other exported + // functions. Because the exported functions are guaranteed to be + // synchronized with Close(). + mu sync.Mutex + closed *grpcsync.Event + done *grpcsync.Event + + bOpts balancer.BuildOptions + logger *grpclog.PrefixLogger + xdsClient client.XDSClient + + config *LBConfig + childLB balancer.Balancer + cancelLoadReport func() + edsServiceName string + lrsServerName *string + loadWrapper *loadstore.Wrapper + + clusterNameMu sync.Mutex + clusterName string + + scWrappersMu sync.Mutex + // The SubConns passed to the child policy are wrapped in a wrapper, to keep + // locality ID. But when the parent ClientConn sends updates, it's going to + // give the original SubConn, not the wrapper. But the child policies only + // know about the wrapper, so when forwarding SubConn updates, they must be + // sent for the wrappers. + // + // This keeps a map from original SubConn to wrapper, so that when + // forwarding the SubConn state update, the child policy will get the + // wrappers. + scWrappers map[balancer.SubConn]*scWrapper + + // childState/drops/requestCounter keeps the state used by the most recently + // generated picker. All fields can only be accessed in run(). And run() is + // the only goroutine that sends picker to the parent ClientConn. All + // requests to update picker need to be sent to pickerUpdateCh. + childState balancer.State + dropCategories []DropConfig // The categories for drops. + drops []*dropper + requestCounterCluster string // The cluster name for the request counter. + requestCounterService string // The service name for the request counter. + requestCounter *client.ClusterRequestsCounter + requestCountMax uint32 + pickerUpdateCh *buffer.Unbounded +} + +// updateLoadStore checks the config for load store, and decides whether it +// needs to restart the load reporting stream. +func (b *clusterImplBalancer) updateLoadStore(newConfig *LBConfig) error { + var updateLoadClusterAndService bool + + // ClusterName is different, restart. ClusterName is from ClusterName and + // EDSServiceName. + clusterName := b.getClusterName() + if clusterName != newConfig.Cluster { + updateLoadClusterAndService = true + b.setClusterName(newConfig.Cluster) + clusterName = newConfig.Cluster + } + if b.edsServiceName != newConfig.EDSServiceName { + updateLoadClusterAndService = true + b.edsServiceName = newConfig.EDSServiceName + } + if updateLoadClusterAndService { + // This updates the clusterName and serviceName that will be reported + // for the loads. The update here is too early, the perfect timing is + // when the picker is updated with the new connection. But from this + // balancer's point of view, it's impossible to tell. + // + // On the other hand, this will almost never happen. Each LRS policy + // shouldn't get updated config. The parent should do a graceful switch + // when the clusterName or serviceName is changed. + b.loadWrapper.UpdateClusterAndService(clusterName, b.edsServiceName) + } + + var ( + stopOldLoadReport bool + startNewLoadReport bool + ) + + // Check if it's necessary to restart load report. + if b.lrsServerName == nil { + if newConfig.LoadReportingServerName != nil { + // Old is nil, new is not nil, start new LRS. + b.lrsServerName = newConfig.LoadReportingServerName + startNewLoadReport = true + } + // Old is nil, new is nil, do nothing. + } else if newConfig.LoadReportingServerName == nil { + // Old is not nil, new is nil, stop old, don't start new. + b.lrsServerName = newConfig.LoadReportingServerName + stopOldLoadReport = true + } else { + // Old is not nil, new is not nil, compare string values, if + // different, stop old and start new. + if *b.lrsServerName != *newConfig.LoadReportingServerName { + b.lrsServerName = newConfig.LoadReportingServerName + stopOldLoadReport = true + startNewLoadReport = true + } + } + + if stopOldLoadReport { + if b.cancelLoadReport != nil { + b.cancelLoadReport() + b.cancelLoadReport = nil + if !startNewLoadReport { + // If a new LRS stream will be started later, no need to update + // it to nil here. + b.loadWrapper.UpdateLoadStore(nil) + } + } + } + if startNewLoadReport { + var loadStore *load.Store + if b.xdsClient != nil { + loadStore, b.cancelLoadReport = b.xdsClient.ReportLoad(*b.lrsServerName) + } + b.loadWrapper.UpdateLoadStore(loadStore) + } + + return nil +} + +func (b *clusterImplBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + if b.closed.HasFired() { + b.logger.Warningf("xds: received ClientConnState {%+v} after clusterImplBalancer was closed", s) + return nil + } + + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + + // Need to check for potential errors at the beginning of this function, so + // that on errors, we reject the whole config, instead of applying part of + // it. + bb := balancer.Get(newConfig.ChildPolicy.Name) + if bb == nil { + return fmt.Errorf("balancer %q not registered", newConfig.ChildPolicy.Name) + } + + if b.xdsClient == nil { + c := client.FromResolverState(s.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + } + + // Update load reporting config. This needs to be done before updating the + // child policy because we need the loadStore from the updated client to be + // passed to the ccWrapper, so that the next picker from the child policy + // will pick up the new loadStore. + if err := b.updateLoadStore(newConfig); err != nil { + return err + } + + // If child policy is a different type, recreate the sub-balancer. + if b.config == nil || b.config.ChildPolicy.Name != newConfig.ChildPolicy.Name { + if b.childLB != nil { + b.childLB.Close() + } + b.childLB = bb.Build(b, b.bOpts) + } + b.config = newConfig + + if b.childLB == nil { + // This is not an expected situation, and should be super rare in + // practice. + // + // When this happens, we already applied all the other configurations + // (drop/circuit breaking), but there's no child policy. This balancer + // will be stuck, and we report the error to the parent. + return fmt.Errorf("child policy is nil, this means balancer %q's Build() returned nil", newConfig.ChildPolicy.Name) + } + + // Notify run() of this new config, in case drop and request counter need + // update (which means a new picker needs to be generated). + b.pickerUpdateCh.Put(newConfig) + + // Addresses and sub-balancer config are sent to sub-balancer. + return b.childLB.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: s.ResolverState, + BalancerConfig: b.config.ChildPolicy.Config, + }) +} + +func (b *clusterImplBalancer) ResolverError(err error) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received resolver error {%+v} after clusterImplBalancer was closed", err) + return + } + + if b.childLB != nil { + b.childLB.ResolverError(err) + } +} + +func (b *clusterImplBalancer) UpdateSubConnState(sc balancer.SubConn, s balancer.SubConnState) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received subconn state change {%+v, %+v} after clusterImplBalancer was closed", sc, s) + return + } + + // Trigger re-resolution when a SubConn turns transient failure. This is + // necessary for the LogicalDNS in cluster_resolver policy to re-resolve. + // + // Note that this happens not only for the addresses from DNS, but also for + // EDS (cluster_impl doesn't know if it's DNS or EDS, only the parent + // knows). The parent priority policy is configured to ignore re-resolution + // signal from the EDS children. + if s.ConnectivityState == connectivity.TransientFailure { + b.ClientConn.ResolveNow(resolver.ResolveNowOptions{}) + } + + b.scWrappersMu.Lock() + if scw, ok := b.scWrappers[sc]; ok { + sc = scw + if s.ConnectivityState == connectivity.Shutdown { + // Remove this SubConn from the map on Shutdown. + delete(b.scWrappers, scw.SubConn) + } + } + b.scWrappersMu.Unlock() + if b.childLB != nil { + b.childLB.UpdateSubConnState(sc, s) + } +} + +func (b *clusterImplBalancer) Close() { + b.mu.Lock() + b.closed.Fire() + b.mu.Unlock() + + if b.childLB != nil { + b.childLB.Close() + b.childLB = nil + } + <-b.done.Done() + b.logger.Infof("Shutdown") +} + +func (b *clusterImplBalancer) ExitIdle() { + if b.childLB == nil { + return + } + if ei, ok := b.childLB.(balancer.ExitIdler); ok { + ei.ExitIdle() + return + } + // Fallback for children that don't support ExitIdle -- connect to all + // SubConns. + for _, sc := range b.scWrappers { + sc.Connect() + } +} + +// Override methods to accept updates from the child LB. + +func (b *clusterImplBalancer) UpdateState(state balancer.State) { + // Instead of updating parent ClientConn inline, send state to run(). + b.pickerUpdateCh.Put(state) +} + +func (b *clusterImplBalancer) setClusterName(n string) { + b.clusterNameMu.Lock() + defer b.clusterNameMu.Unlock() + b.clusterName = n +} + +func (b *clusterImplBalancer) getClusterName() string { + b.clusterNameMu.Lock() + defer b.clusterNameMu.Unlock() + return b.clusterName +} + +// scWrapper is a wrapper of SubConn with locality ID. The locality ID can be +// retrieved from the addresses when creating SubConn. +// +// All SubConns passed to the child policies are wrapped in this, so that the +// picker can get the localityID from the picked SubConn, and do load reporting. +// +// After wrapping, all SubConns to and from the parent ClientConn (e.g. for +// SubConn state update, update/remove SubConn) must be the original SubConns. +// All SubConns to and from the child policy (NewSubConn, forwarding SubConn +// state update) must be the wrapper. The balancer keeps a map from the original +// SubConn to the wrapper for this purpose. +type scWrapper struct { + balancer.SubConn + // locality needs to be atomic because it can be updated while being read by + // the picker. + locality atomic.Value // type resource.LocalityID +} + +func (scw *scWrapper) updateLocalityID(lID resource.LocalityID) { + scw.locality.Store(lID) +} + +func (scw *scWrapper) localityID() resource.LocalityID { + lID, _ := scw.locality.Load().(resource.LocalityID) + return lID +} + +func (b *clusterImplBalancer) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + clusterName := b.getClusterName() + newAddrs := make([]resolver.Address, len(addrs)) + var lID resource.LocalityID + for i, addr := range addrs { + newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName) + lID = resource.GetLocalityID(newAddrs[i]) + } + sc, err := b.ClientConn.NewSubConn(newAddrs, opts) + if err != nil { + return nil, err + } + // Wrap this SubConn in a wrapper, and add it to the map. + b.scWrappersMu.Lock() + ret := &scWrapper{SubConn: sc} + ret.updateLocalityID(lID) + b.scWrappers[sc] = ret + b.scWrappersMu.Unlock() + return ret, nil +} + +func (b *clusterImplBalancer) RemoveSubConn(sc balancer.SubConn) { + scw, ok := sc.(*scWrapper) + if !ok { + b.ClientConn.RemoveSubConn(sc) + return + } + // Remove the original SubConn from the parent ClientConn. + // + // Note that we don't remove this SubConn from the scWrappers map. We will + // need it to forward the final SubConn state Shutdown to the child policy. + // + // This entry is kept in the map until it's state is changes to Shutdown, + // and will be deleted in UpdateSubConnState(). + b.ClientConn.RemoveSubConn(scw.SubConn) +} + +func (b *clusterImplBalancer) UpdateAddresses(sc balancer.SubConn, addrs []resolver.Address) { + clusterName := b.getClusterName() + newAddrs := make([]resolver.Address, len(addrs)) + var lID resource.LocalityID + for i, addr := range addrs { + newAddrs[i] = internal.SetXDSHandshakeClusterName(addr, clusterName) + lID = resource.GetLocalityID(newAddrs[i]) + } + if scw, ok := sc.(*scWrapper); ok { + scw.updateLocalityID(lID) + // Need to get the original SubConn from the wrapper before calling + // parent ClientConn. + sc = scw.SubConn + } + b.ClientConn.UpdateAddresses(sc, newAddrs) +} + +type dropConfigs struct { + drops []*dropper + requestCounter *client.ClusterRequestsCounter + requestCountMax uint32 +} + +// handleDropAndRequestCount compares drop and request counter in newConfig with +// the one currently used by picker. It returns a new dropConfigs if a new +// picker needs to be generated, otherwise it returns nil. +func (b *clusterImplBalancer) handleDropAndRequestCount(newConfig *LBConfig) *dropConfigs { + // Compare new drop config. And update picker if it's changed. + var updatePicker bool + if !equalDropCategories(b.dropCategories, newConfig.DropCategories) { + b.dropCategories = newConfig.DropCategories + b.drops = make([]*dropper, 0, len(newConfig.DropCategories)) + for _, c := range newConfig.DropCategories { + b.drops = append(b.drops, newDropper(c)) + } + updatePicker = true + } + + // Compare cluster name. And update picker if it's changed, because circuit + // breaking's stream counter will be different. + if b.requestCounterCluster != newConfig.Cluster || b.requestCounterService != newConfig.EDSServiceName { + b.requestCounterCluster = newConfig.Cluster + b.requestCounterService = newConfig.EDSServiceName + b.requestCounter = client.GetClusterRequestsCounter(newConfig.Cluster, newConfig.EDSServiceName) + updatePicker = true + } + // Compare upper bound of stream count. And update picker if it's changed. + // This is also for circuit breaking. + var newRequestCountMax uint32 = 1024 + if newConfig.MaxConcurrentRequests != nil { + newRequestCountMax = *newConfig.MaxConcurrentRequests + } + if b.requestCountMax != newRequestCountMax { + b.requestCountMax = newRequestCountMax + updatePicker = true + } + + if !updatePicker { + return nil + } + return &dropConfigs{ + drops: b.drops, + requestCounter: b.requestCounter, + requestCountMax: b.requestCountMax, + } +} + +func (b *clusterImplBalancer) run() { + defer b.done.Fire() + for { + select { + case update := <-b.pickerUpdateCh.Get(): + b.pickerUpdateCh.Load() + b.mu.Lock() + if b.closed.HasFired() { + b.mu.Unlock() + return + } + switch u := update.(type) { + case balancer.State: + b.childState = u + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: newPicker(b.childState, &dropConfigs{ + drops: b.drops, + requestCounter: b.requestCounter, + requestCountMax: b.requestCountMax, + }, b.loadWrapper), + }) + case *LBConfig: + dc := b.handleDropAndRequestCount(u) + if dc != nil && b.childState.Picker != nil { + b.ClientConn.UpdateState(balancer.State{ + ConnectivityState: b.childState.ConnectivityState, + Picker: newPicker(b.childState, dc, b.loadWrapper), + }) + } + } + b.mu.Unlock() + case <-b.closed.Done(): + if b.cancelLoadReport != nil { + b.cancelLoadReport() + b.cancelLoadReport = nil + } + return + } + } +} diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go new file mode 100644 index 0000000000..a33ef0472a --- /dev/null +++ b/xds/balancer/clusterimpl/config.go @@ -0,0 +1,64 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "encoding/json" + + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// DropConfig contains the category, and drop ratio. +type DropConfig struct { + Category string + RequestsPerMillion uint32 +} + +// LBConfig is the balancer config for cluster_impl balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Cluster string `json:"cluster,omitempty"` + EDSServiceName string `json:"edsServiceName,omitempty"` + LoadReportingServerName *string `json:"lrsLoadReportingServerName,omitempty"` + MaxConcurrentRequests *uint32 `json:"maxConcurrentRequests,omitempty"` + DropCategories []DropConfig `json:"dropCategories,omitempty"` + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} + +func equalDropCategories(a, b []DropConfig) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go new file mode 100644 index 0000000000..b217a32119 --- /dev/null +++ b/xds/balancer/clusterimpl/config_test.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "testing" + + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + _ "google.golang.org/grpc/balancer/roundrobin" + _ "google.golang.org/grpc/balancer/weightedtarget" +) + +const ( + testJSONConfig = `{ + "cluster": "test_cluster", + "edsServiceName": "test-eds", + "lrsLoadReportingServerName": "lrs_server", + "maxConcurrentRequests": 123, + "dropCategories": [ + { + "category": "drop-1", + "requestsPerMillion": 314 + }, + { + "category": "drop-2", + "requestsPerMillion": 159 + } + ], + "childPolicy": [ + { + "weighted_target_experimental": { + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } + } + } + ] +}` + + wtName = "weighted_target_experimental" +) + +var ( + wtConfigParser = balancer.Get(wtName).(balancer.ConfigParser) + wtConfigJSON = `{ + "targets": { + "wt-child-1": { + "weight": 75, + "childPolicy":[{"round_robin":{}}] + }, + "wt-child-2": { + "weight": 25, + "childPolicy":[{"round_robin":{}}] + } + } +}` + + wtConfig, _ = wtConfigParser.ParseConfig([]byte(wtConfigJSON)) +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "empty json", + js: "", + want: nil, + wantErr: true, + }, + { + name: "bad json", + js: "{", + want: nil, + wantErr: true, + }, + { + name: "OK", + js: testJSONConfig, + want: &LBConfig{ + Cluster: "test_cluster", + EDSServiceName: "test-eds", + LoadReportingServerName: newString("lrs_server"), + MaxConcurrentRequests: newUint32(123), + DropCategories: []DropConfig{ + {Category: "drop-1", RequestsPerMillion: 314}, + {Category: "drop-2", RequestsPerMillion: 159}, + }, + ChildPolicy: &internalserviceconfig.BalancerConfig{ + Name: wtName, + Config: wtConfig, + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Fatalf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + } + if !cmp.Equal(got, tt.want) { + t.Errorf("parseConfig() got unexpected result, diff: %v", cmp.Diff(got, tt.want)) + } + }) + } +} + +func newString(s string) *string { + return &s +} + +func newUint32(i uint32) *uint32 { + return &i +} diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go new file mode 100644 index 0000000000..70c8d2dad1 --- /dev/null +++ b/xds/balancer/clusterimpl/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[xds-cluster-impl-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clusterImplBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go new file mode 100644 index 0000000000..5e3da761fd --- /dev/null +++ b/xds/balancer/clusterimpl/picker.go @@ -0,0 +1,189 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterimpl + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" + orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" +) + +// NewRandomWRR is used when calculating drops. It's exported so that tests can +// override it. +var NewRandomWRR = wrr.NewRandom + +const million = 1000000 + +type dropper struct { + category string + w wrr.WRR +} + +// greatest common divisor (GCD) via Euclidean algorithm +func gcd(a, b uint32) uint32 { + for b != 0 { + t := b + b = a % b + a = t + } + return a +} + +func newDropper(c DropConfig) *dropper { + w := NewRandomWRR() + gcdv := gcd(c.RequestsPerMillion, million) + // Return true for RequestPerMillion, false for the rest. + w.Add(true, int64(c.RequestsPerMillion/gcdv)) + w.Add(false, int64((million-c.RequestsPerMillion)/gcdv)) + + return &dropper{ + category: c.Category, + w: w, + } +} + +func (d *dropper) drop() (ret bool) { + return d.w.Next().(bool) +} + +const ( + serverLoadCPUName = "cpu_utilization" + serverLoadMemoryName = "mem_utilization" +) + +// loadReporter wraps the methods from the loadStore that are used here. +type loadReporter interface { + CallStarted(locality string) + CallFinished(locality string, err error) + CallServerLoad(locality, name string, val float64) + CallDropped(locality string) +} + +// Picker implements RPC drop, circuit breaking drop and load reporting. +type picker struct { + drops []*dropper + s balancer.State + loadStore loadReporter + counter *client.ClusterRequestsCounter + countMax uint32 +} + +func newPicker(s balancer.State, config *dropConfigs, loadStore load.PerClusterReporter) *picker { + return &picker{ + drops: config.drops, + s: s, + loadStore: loadStore, + counter: config.requestCounter, + countMax: config.requestCountMax, + } +} + +func (d *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + // Don't drop unless the inner picker is READY. Similar to + // https://github.com/grpc/grpc-go/issues/2622. + if d.s.ConnectivityState == connectivity.Ready { + // Check if this RPC should be dropped by category. + for _, dp := range d.drops { + if dp.drop() { + if d.loadStore != nil { + d.loadStore.CallDropped(dp.category) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "RPC is dropped") + } + } + } + + // Check if this RPC should be dropped by circuit breaking. + if d.counter != nil { + if err := d.counter.StartRequest(d.countMax); err != nil { + // Drops by circuit breaking are reported with empty category. They + // will be reported only in total drops, but not in per category. + if d.loadStore != nil { + d.loadStore.CallDropped("") + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, err.Error()) + } + } + + var lIDStr string + pr, err := d.s.Picker.Pick(info) + if scw, ok := pr.SubConn.(*scWrapper); ok { + // This OK check also covers the case err!=nil, because SubConn will be + // nil. + pr.SubConn = scw.SubConn + var e error + // If locality ID isn't found in the wrapper, an empty locality ID will + // be used. + lIDStr, e = scw.localityID().ToString() + if e != nil { + logger.Infof("failed to marshal LocalityID: %#v, loads won't be reported", scw.localityID()) + } + } + + if err != nil { + if d.counter != nil { + // Release one request count if this pick fails. + d.counter.EndRequest() + } + return pr, err + } + + if d.loadStore != nil { + d.loadStore.CallStarted(lIDStr) + oldDone := pr.Done + pr.Done = func(info balancer.DoneInfo) { + if oldDone != nil { + oldDone(info) + } + d.loadStore.CallFinished(lIDStr, info.Err) + + load, ok := info.ServerLoad.(*orcapb.OrcaLoadReport) + if !ok { + return + } + d.loadStore.CallServerLoad(lIDStr, serverLoadCPUName, load.CpuUtilization) + d.loadStore.CallServerLoad(lIDStr, serverLoadMemoryName, load.MemUtilization) + for n, c := range load.RequestCost { + d.loadStore.CallServerLoad(lIDStr, n, c) + } + for n, c := range load.Utilization { + d.loadStore.CallServerLoad(lIDStr, n, c) + } + } + } + + if d.counter != nil { + // Update Done() so that when the RPC finishes, the request count will + // be released. + oldDone := pr.Done + pr.Done = func(doneInfo balancer.DoneInfo) { + d.counter.EndRequest() + if oldDone != nil { + oldDone(doneInfo) + } + } + } + + return pr, err +} diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go new file mode 100644 index 0000000000..aec4fb658c --- /dev/null +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -0,0 +1,222 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "fmt" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" +) + +type subBalancerState struct { + state balancer.State + // stateToAggregate is the connectivity state used only for state + // aggregation. It could be different from state.ConnectivityState. For + // example when a sub-balancer transitions from TransientFailure to + // connecting, state.ConnectivityState is Connecting, but stateToAggregate + // is still TransientFailure. + stateToAggregate connectivity.State +} + +func (s *subBalancerState) String() string { + return fmt.Sprintf("picker:%p,state:%v,stateToAggregate:%v", s.state.Picker, s.state.ConnectivityState, s.stateToAggregate) +} + +type balancerStateAggregator struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + + mu sync.Mutex + // If started is false, no updates should be sent to the parent cc. A closed + // sub-balancer could still send pickers to this aggregator. This makes sure + // that no updates will be forwarded to parent when the whole balancer group + // and states aggregator is closed. + started bool + // All balancer IDs exist as keys in this map, even if balancer group is not + // started. + // + // If an ID is not in map, it's either removed or never added. + idToPickerState map[string]*subBalancerState +} + +func newBalancerStateAggregator(cc balancer.ClientConn, logger *grpclog.PrefixLogger) *balancerStateAggregator { + return &balancerStateAggregator{ + cc: cc, + logger: logger, + idToPickerState: make(map[string]*subBalancerState), + } +} + +// Start starts the aggregator. It can be called after Close to restart the +// aggretator. +func (bsa *balancerStateAggregator) start() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.started = true +} + +// Close closes the aggregator. When the aggregator is closed, it won't call +// parent ClientConn to update balancer state. +func (bsa *balancerStateAggregator) close() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.started = false + bsa.clearStates() +} + +// add adds a sub-balancer state with weight. It adds a place holder, and waits +// for the real sub-balancer to update state. +// +// This is called when there's a new child. +func (bsa *balancerStateAggregator) add(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + bsa.idToPickerState[id] = &subBalancerState{ + // Start everything in CONNECTING, so if one of the sub-balancers + // reports TransientFailure, the RPCs will still wait for the other + // sub-balancers. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + stateToAggregate: connectivity.Connecting, + } +} + +// remove removes the sub-balancer state. Future updates from this sub-balancer, +// if any, will be ignored. +// +// This is called when a child is removed. +func (bsa *balancerStateAggregator) remove(id string) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + if _, ok := bsa.idToPickerState[id]; !ok { + return + } + // Remove id and picker from picker map. This also results in future updates + // for this ID to be ignored. + delete(bsa.idToPickerState, id) +} + +// UpdateState is called to report a balancer state change from sub-balancer. +// It's usually called by the balancer group. +// +// It calls parent ClientConn's UpdateState with the new aggregated state. +func (bsa *balancerStateAggregator) UpdateState(id string, state balancer.State) { + bsa.mu.Lock() + defer bsa.mu.Unlock() + pickerSt, ok := bsa.idToPickerState[id] + if !ok { + // All state starts with an entry in pickStateMap. If ID is not in map, + // it's either removed, or never existed. + return + } + if !(pickerSt.state.ConnectivityState == connectivity.TransientFailure && state.ConnectivityState == connectivity.Connecting) { + // If old state is TransientFailure, and new state is Connecting, don't + // update the state, to prevent the aggregated state from being always + // CONNECTING. Otherwise, stateToAggregate is the same as + // state.ConnectivityState. + pickerSt.stateToAggregate = state.ConnectivityState + } + pickerSt.state = state + + if !bsa.started { + return + } + bsa.cc.UpdateState(bsa.build()) +} + +// clearState Reset everything to init state (Connecting) but keep the entry in +// map (to keep the weight). +// +// Caller must hold bsa.mu. +func (bsa *balancerStateAggregator) clearStates() { + for _, pState := range bsa.idToPickerState { + pState.state = balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + } + pState.stateToAggregate = connectivity.Connecting + } +} + +// buildAndUpdate combines the sub-state from each sub-balancer into one state, +// and update it to parent ClientConn. +func (bsa *balancerStateAggregator) buildAndUpdate() { + bsa.mu.Lock() + defer bsa.mu.Unlock() + if !bsa.started { + return + } + bsa.cc.UpdateState(bsa.build()) +} + +// build combines sub-states into one. The picker will do a child pick. +// +// Caller must hold bsa.mu. +func (bsa *balancerStateAggregator) build() balancer.State { + // TODO: the majority of this function (and UpdateState) is exactly the same + // as weighted_target's state aggregator. Try to make a general utility + // function/struct to handle the logic. + // + // One option: make a SubBalancerState that handles Update(State), including + // handling the special connecting after ready, as in UpdateState(). Then a + // function to calculate the aggregated connectivity state as in this + // function. + // + // TODO: use balancer.ConnectivityStateEvaluator to calculate the aggregated + // state. + var readyN, connectingN, idleN int + for _, ps := range bsa.idToPickerState { + switch ps.stateToAggregate { + case connectivity.Ready: + readyN++ + case connectivity.Connecting: + connectingN++ + case connectivity.Idle: + idleN++ + } + } + var aggregatedState connectivity.State + switch { + case readyN > 0: + aggregatedState = connectivity.Ready + case connectingN > 0: + aggregatedState = connectivity.Connecting + case idleN > 0: + aggregatedState = connectivity.Idle + default: + aggregatedState = connectivity.TransientFailure + } + + // The picker's return error might not be consistent with the + // aggregatedState. Because for this LB policy, we want to always build + // picker with all sub-pickers (not only ready sub-pickers), so even if the + // overall state is Ready, pick for certain RPCs can behave like Connecting + // or TransientFailure. + bsa.logger.Infof("Child pickers: %+v", bsa.idToPickerState) + return balancer.State{ + ConnectivityState: aggregatedState, + Picker: newPickerGroup(bsa.idToPickerState), + } +} diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go new file mode 100644 index 0000000000..738fba1bc7 --- /dev/null +++ b/xds/balancer/clustermanager/clustermanager.go @@ -0,0 +1,149 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clustermanager implements the cluster manager LB policy for xds. +package clustermanager + +import ( + "encoding/json" + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +const balancerName = "xds_cluster_manager_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &bal{} + b.logger = prefixLogger(b) + b.stateAggregator = newBalancerStateAggregator(cc, b.logger) + b.stateAggregator.start() + b.bg = balancergroup.New(cc, opts, b.stateAggregator, b.logger) + b.bg.Start() + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return balancerName +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type bal struct { + logger *internalgrpclog.PrefixLogger + + // TODO: make this package not dependent on xds specific code. Same as for + // weighted target balancer. + bg *balancergroup.BalancerGroup + stateAggregator *balancerStateAggregator + + children map[string]childConfig +} + +func (b *bal) updateChildren(s balancer.ClientConnState, newConfig *lbConfig) { + update := false + addressesSplit := hierarchy.Group(s.ResolverState.Addresses) + + // Remove sub-pickers and sub-balancers that are not in the new cluster list. + for name := range b.children { + if _, ok := newConfig.Children[name]; !ok { + b.stateAggregator.remove(name) + b.bg.Remove(name) + update = true + } + } + + // For sub-balancers in the new cluster list, + // - add to balancer group if it's new, + // - forward the address/balancer config update. + for name, newT := range newConfig.Children { + if _, ok := b.children[name]; !ok { + // If this is a new sub-balancer, add it to the picker map. + b.stateAggregator.add(name) + // Then add to the balancer group. + b.bg.Add(name, balancer.Get(newT.ChildPolicy.Name)) + } + // TODO: handle error? How to aggregate errors and return? + _ = b.bg.UpdateClientConnState(name, balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }, + BalancerConfig: newT.ChildPolicy.Config, + }) + } + + b.children = newConfig.Children + if update { + b.stateAggregator.buildAndUpdate() + } +} + +func (b *bal) UpdateClientConnState(s balancer.ClientConnState) error { + newConfig, ok := s.BalancerConfig.(*lbConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + b.logger.Infof("update with config %+v, resolver state %+v", pretty.ToJSON(s.BalancerConfig), s.ResolverState) + + b.updateChildren(s, newConfig) + return nil +} + +func (b *bal) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *bal) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.bg.UpdateSubConnState(sc, state) +} + +func (b *bal) Close() { + b.stateAggregator.close() + b.bg.Close() + b.logger.Infof("Shutdown") +} + +func (b *bal) ExitIdle() { + b.bg.ExitIdle() +} + +const prefix = "[xds-cluster-manager-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *bal) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go new file mode 100644 index 0000000000..f04d78ed51 --- /dev/null +++ b/xds/balancer/clustermanager/config.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "encoding/json" + + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +type childConfig struct { + // ChildPolicy is the child policy and it's config. + ChildPolicy *internalserviceconfig.BalancerConfig +} + +// lbConfig is the balancer config for xds routing policy. +type lbConfig struct { + serviceconfig.LoadBalancingConfig + Children map[string]childConfig +} + +func parseConfig(c json.RawMessage) (*lbConfig, error) { + cfg := &lbConfig{} + if err := json.Unmarshal(c, cfg); err != nil { + return nil, err + } + + return cfg, nil +} diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go new file mode 100644 index 0000000000..015cd2b7af --- /dev/null +++ b/xds/balancer/clustermanager/picker.go @@ -0,0 +1,70 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clustermanager + +import ( + "context" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// pickerGroup contains a list of pickers. If the picker isn't ready, the pick +// will be queued. +type pickerGroup struct { + pickers map[string]balancer.Picker +} + +func newPickerGroup(idToPickerState map[string]*subBalancerState) *pickerGroup { + pickers := make(map[string]balancer.Picker) + for id, st := range idToPickerState { + pickers[id] = st.state.Picker + } + return &pickerGroup{ + pickers: pickers, + } +} + +func (pg *pickerGroup) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + cluster := getPickedCluster(info.Ctx) + if p := pg.pickers[cluster]; p != nil { + return p.Pick(info) + } + return balancer.PickResult{}, status.Errorf(codes.Unavailable, "unknown cluster selected for RPC: %q", cluster) +} + +type clusterKey struct{} + +func getPickedCluster(ctx context.Context) string { + cluster, _ := ctx.Value(clusterKey{}).(string) + return cluster +} + +// GetPickedClusterForTesting returns the cluster in the context; to be used +// for testing only. +func GetPickedClusterForTesting(ctx context.Context) string { + return getPickedCluster(ctx) +} + +// SetPickedCluster adds the selected cluster to the context for the +// xds_cluster_manager LB policy to pick. +func SetPickedCluster(ctx context.Context, cluster string) context.Context { + return context.WithValue(ctx, clusterKey{}, cluster) +} diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go new file mode 100644 index 0000000000..45b1f6b51c --- /dev/null +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -0,0 +1,379 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterresolver contains EDS balancer implementation. +package clusterresolver + +import ( + "encoding/json" + "errors" + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the cluster_resolver balancer. +const Name = "cluster_resolver_experimental" + +var ( + errBalancerClosed = errors.New("cdsBalancer is closed") + newChildBalancer = func(bb balancer.Builder, cc balancer.ClientConn, o balancer.BuildOptions) balancer.Balancer { + return bb.Build(cc, o) + } +) + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +// Build helps implement the balancer.Builder interface. +func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + priorityBuilder := balancer.Get(priority.Name) + if priorityBuilder == nil { + logger.Errorf("priority balancer is needed but not registered") + return nil + } + priorityConfigParser, ok := priorityBuilder.(balancer.ConfigParser) + if !ok { + logger.Errorf("priority balancer builder is not a config parser") + return nil + } + + b := &clusterResolverBalancer{ + bOpts: opts, + updateCh: buffer.NewUnbounded(), + closed: grpcsync.NewEvent(), + done: grpcsync.NewEvent(), + + priorityBuilder: priorityBuilder, + priorityConfigParser: priorityConfigParser, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + + b.resourceWatcher = newResourceResolver(b) + b.cc = &ccWrapper{ + ClientConn: cc, + resourceWatcher: b.resourceWatcher, + } + + go b.run() + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, fmt.Errorf("unable to unmarshal balancer config %s into cluster-resolver config, error: %v", string(c), err) + } + return &cfg, nil +} + +// ccUpdate wraps a clientConn update received from gRPC (pushed from the +// xdsResolver). +type ccUpdate struct { + state balancer.ClientConnState + err error +} + +// scUpdate wraps a subConn update received from gRPC. This is directly passed +// on to the child balancer. +type scUpdate struct { + subConn balancer.SubConn + state balancer.SubConnState +} + +type exitIdle struct{} + +// clusterResolverBalancer manages xdsClient and the actual EDS balancer implementation that +// does load balancing. +// +// It currently has only an clusterResolverBalancer. Later, we may add fallback. +type clusterResolverBalancer struct { + cc balancer.ClientConn + bOpts balancer.BuildOptions + updateCh *buffer.Unbounded // Channel for updates from gRPC. + resourceWatcher *resourceResolver + logger *grpclog.PrefixLogger + closed *grpcsync.Event + done *grpcsync.Event + + priorityBuilder balancer.Builder + priorityConfigParser balancer.ConfigParser + + config *LBConfig + configRaw *serviceconfig.ParseResult + xdsClient client.XDSClient // xDS client to watch EDS resource. + attrsWithClient *attributes.Attributes // Attributes with xdsClient attached to be passed to the child policies. + + child balancer.Balancer + priorities []priorityConfig + watchUpdateReceived bool +} + +// handleClientConnUpdate handles a ClientConnUpdate received from gRPC. Good +// updates lead to registration of EDS and DNS watches. Updates with error lead +// to cancellation of existing watch and propagation of the same error to the +// child balancer. +func (b *clusterResolverBalancer) handleClientConnUpdate(update *ccUpdate) { + // We first handle errors, if any, and then proceed with handling the + // update, only if the status quo has changed. + if err := update.err; err != nil { + b.handleErrorFromUpdate(err, true) + return + } + + b.logger.Infof("Receive update from resolver, balancer config: %v", pretty.ToJSON(update.state.BalancerConfig)) + cfg, _ := update.state.BalancerConfig.(*LBConfig) + if cfg == nil { + b.logger.Warningf("xds: unexpected LoadBalancingConfig type: %T", update.state.BalancerConfig) + return + } + + b.config = cfg + b.configRaw = update.state.ResolverState.ServiceConfig + b.resourceWatcher.updateMechanisms(cfg.DiscoveryMechanisms) + + if !b.watchUpdateReceived { + // If update was not received, wait for it. + return + } + // If eds resp was received before this, the child policy was created. We + // need to generate a new balancer config and send it to the child, because + // certain fields (unrelated to EDS watch) might have changed. + if err := b.updateChildConfig(); err != nil { + b.logger.Warningf("failed to update child policy config: %v", err) + } +} + +// handleWatchUpdate handles a watch update from the xDS Client. Good updates +// lead to clientConn updates being invoked on the underlying child balancer. +func (b *clusterResolverBalancer) handleWatchUpdate(update *resourceUpdate) { + if err := update.err; err != nil { + b.logger.Warningf("Watch error from xds-client %p: %v", b.xdsClient, err) + b.handleErrorFromUpdate(err, false) + return + } + + b.logger.Infof("resource update: %+v", pretty.ToJSON(update.priorities)) + b.watchUpdateReceived = true + b.priorities = update.priorities + + // A new EDS update triggers new child configs (e.g. different priorities + // for the priority balancer), and new addresses (the endpoints come from + // the EDS response). + if err := b.updateChildConfig(); err != nil { + b.logger.Warningf("failed to update child policy's balancer config: %v", err) + } +} + +// updateChildConfig builds a balancer config from eb's cached eds resp and +// service config, and sends that to the child balancer. Note that it also +// generates the addresses, because the endpoints come from the EDS resp. +// +// If child balancer doesn't already exist, one will be created. +func (b *clusterResolverBalancer) updateChildConfig() error { + // Child was build when the first EDS resp was received, so we just build + // the config and addresses. + if b.child == nil { + b.child = newChildBalancer(b.priorityBuilder, b.cc, b.bOpts) + } + + childCfgBytes, addrs, err := buildPriorityConfigJSON(b.priorities, b.config.XDSLBPolicy) + if err != nil { + return fmt.Errorf("failed to build priority balancer config: %v", err) + } + childCfg, err := b.priorityConfigParser.ParseConfig(childCfgBytes) + if err != nil { + return fmt.Errorf("failed to parse generated priority balancer config, this should never happen because the config is generated: %v", err) + } + b.logger.Infof("build balancer config: %v", pretty.ToJSON(childCfg)) + return b.child.UpdateClientConnState(balancer.ClientConnState{ + ResolverState: resolver.State{ + Addresses: addrs, + ServiceConfig: b.configRaw, + Attributes: b.attrsWithClient, + }, + BalancerConfig: childCfg, + }) +} + +// handleErrorFromUpdate handles both the error from parent ClientConn (from CDS +// balancer) and the error from xds client (from the watcher). fromParent is +// true if error is from parent ClientConn. +// +// If the error is connection error, it should be handled for fallback purposes. +// +// If the error is resource-not-found: +// - If it's from CDS balancer (shows as a resolver error), it means LDS or CDS +// resources were removed. The EDS watch should be canceled. +// - If it's from xds client, it means EDS resource were removed. The EDS +// watcher should keep watching. +// In both cases, the sub-balancers will be receive the error. +func (b *clusterResolverBalancer) handleErrorFromUpdate(err error, fromParent bool) { + b.logger.Warningf("Received error: %v", err) + if fromParent && resource.ErrType(err) == resource.ErrorTypeResourceNotFound { + // This is an error from the parent ClientConn (can be the parent CDS + // balancer), and is a resource-not-found error. This means the resource + // (can be either LDS or CDS) was removed. Stop the EDS watch. + b.resourceWatcher.stop() + } + if b.child != nil { + b.child.ResolverError(err) + } else { + // If eds balancer was never created, fail the RPCs with errors. + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(err), + }) + } + +} + +// run is a long-running goroutine which handles all updates from gRPC and +// client. All methods which are invoked directly by gRPC or xdsClient simply +// push an update onto a channel which is read and acted upon right here. +func (b *clusterResolverBalancer) run() { + for { + select { + case u := <-b.updateCh.Get(): + b.updateCh.Load() + switch update := u.(type) { + case *ccUpdate: + b.handleClientConnUpdate(update) + case *scUpdate: + // SubConn updates are simply handed over to the underlying + // child balancer. + if b.child == nil { + b.logger.Errorf("xds: received scUpdate {%+v} with no child balancer", update) + break + } + b.child.UpdateSubConnState(update.subConn, update.state) + case exitIdle: + if b.child == nil { + b.logger.Errorf("xds: received ExitIdle with no child balancer") + break + } + // This implementation assumes the child balancer supports + // ExitIdle (but still checks for the interface's existence to + // avoid a panic if not). If the child does not, no subconns + // will be connected. + if ei, ok := b.child.(balancer.ExitIdler); ok { + ei.ExitIdle() + } + } + case u := <-b.resourceWatcher.updateChannel: + b.handleWatchUpdate(u) + + // Close results in cancellation of the EDS watch and closing of the + // underlying child policy and is the only way to exit this goroutine. + case <-b.closed.Done(): + b.resourceWatcher.stop() + + if b.child != nil { + b.child.Close() + b.child = nil + } + // This is the *ONLY* point of return from this function. + b.logger.Infof("Shutdown") + b.done.Fire() + return + } + } +} + +// Following are methods to implement the balancer interface. + +// UpdateClientConnState receives the serviceConfig (which contains the +// clusterName to watch for in CDS) and the xdsClient object from the +// xdsResolver. +func (b *clusterResolverBalancer) UpdateClientConnState(state balancer.ClientConnState) error { + if b.closed.HasFired() { + b.logger.Warningf("xds: received ClientConnState {%+v} after clusterResolverBalancer was closed", state) + return errBalancerClosed + } + + if b.xdsClient == nil { + c := client.FromResolverState(state.ResolverState) + if c == nil { + return balancer.ErrBadResolverState + } + b.xdsClient = c + b.attrsWithClient = state.ResolverState.Attributes + } + + b.updateCh.Put(&ccUpdate{state: state}) + return nil +} + +// ResolverError handles errors reported by the xdsResolver. +func (b *clusterResolverBalancer) ResolverError(err error) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received resolver error {%v} after clusterResolverBalancer was closed", err) + return + } + b.updateCh.Put(&ccUpdate{err: err}) +} + +// UpdateSubConnState handles subConn updates from gRPC. +func (b *clusterResolverBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + if b.closed.HasFired() { + b.logger.Warningf("xds: received subConn update {%v, %v} after clusterResolverBalancer was closed", sc, state) + return + } + b.updateCh.Put(&scUpdate{subConn: sc, state: state}) +} + +// Close closes the cdsBalancer and the underlying child balancer. +func (b *clusterResolverBalancer) Close() { + b.closed.Fire() + <-b.done.Done() +} + +func (b *clusterResolverBalancer) ExitIdle() { + b.updateCh.Put(exitIdle{}) +} + +// ccWrapper overrides ResolveNow(), so that re-resolution from the child +// policies will trigger the DNS resolver in cluster_resolver balancer. +type ccWrapper struct { + balancer.ClientConn + resourceWatcher *resourceResolver +} + +func (c *ccWrapper) ResolveNow(resolver.ResolveNowOptions) { + c.resourceWatcher.resolveNow() +} diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go new file mode 100644 index 0000000000..5aa1594104 --- /dev/null +++ b/xds/balancer/clusterresolver/config.go @@ -0,0 +1,185 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package clusterresolver + +import ( + "bytes" + "encoding/json" + "fmt" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/serviceconfig" +) + +// DiscoveryMechanismType is the type of discovery mechanism. +type DiscoveryMechanismType int + +const ( + // DiscoveryMechanismTypeEDS is eds. + DiscoveryMechanismTypeEDS DiscoveryMechanismType = iota // `json:"EDS"` + // DiscoveryMechanismTypeLogicalDNS is DNS. + DiscoveryMechanismTypeLogicalDNS // `json:"LOGICAL_DNS"` +) + +// MarshalJSON marshals a DiscoveryMechanismType to a quoted json string. +// +// This is necessary to handle enum (as strings) from JSON. +// +// Note that this needs to be defined on the type not pointer, otherwise the +// variables of this type will marshal to int not string. +func (t DiscoveryMechanismType) MarshalJSON() ([]byte, error) { + buffer := bytes.NewBufferString(`"`) + switch t { + case DiscoveryMechanismTypeEDS: + buffer.WriteString("EDS") + case DiscoveryMechanismTypeLogicalDNS: + buffer.WriteString("LOGICAL_DNS") + } + buffer.WriteString(`"`) + return buffer.Bytes(), nil +} + +// UnmarshalJSON unmarshals a quoted json string to the DiscoveryMechanismType. +func (t *DiscoveryMechanismType) UnmarshalJSON(b []byte) error { + var s string + err := json.Unmarshal(b, &s) + if err != nil { + return err + } + switch s { + case "EDS": + *t = DiscoveryMechanismTypeEDS + case "LOGICAL_DNS": + *t = DiscoveryMechanismTypeLogicalDNS + default: + return fmt.Errorf("unable to unmarshal string %q to type DiscoveryMechanismType", s) + } + return nil +} + +// DiscoveryMechanism is the discovery mechanism, can be either EDS or DNS. +// +// For DNS, the ClientConn target will be used for name resolution. +// +// For EDS, if EDSServiceName is not empty, it will be used for watching. If +// EDSServiceName is empty, Cluster will be used. +type DiscoveryMechanism struct { + // Cluster is the cluster name. + Cluster string `json:"cluster,omitempty"` + // LoadReportingServerName is the LRS server to send load reports to. If + // not present, load reporting will be disabled. If set to the empty string, + // load reporting will be sent to the same server that we obtained CDS data + // from. + LoadReportingServerName *string `json:"lrsLoadReportingServerName,omitempty"` + // MaxConcurrentRequests is the maximum number of outstanding requests can + // be made to the upstream cluster. Default is 1024. + MaxConcurrentRequests *uint32 `json:"maxConcurrentRequests,omitempty"` + // Type is the discovery mechanism type. + Type DiscoveryMechanismType `json:"type,omitempty"` + // EDSServiceName is the EDS service name, as returned in CDS. May be unset + // if not specified in CDS. For type EDS only. + // + // This is used for EDS watch if set. If unset, Cluster is used for EDS + // watch. + EDSServiceName string `json:"edsServiceName,omitempty"` + // DNSHostname is the DNS name to resolve in "host:port" form. For type + // LOGICAL_DNS only. + DNSHostname string `json:"dnsHostname,omitempty"` +} + +// Equal returns whether the DiscoveryMechanism is the same with the parameter. +func (dm DiscoveryMechanism) Equal(b DiscoveryMechanism) bool { + switch { + case dm.Cluster != b.Cluster: + return false + case !equalStringP(dm.LoadReportingServerName, b.LoadReportingServerName): + return false + case !equalUint32P(dm.MaxConcurrentRequests, b.MaxConcurrentRequests): + return false + case dm.Type != b.Type: + return false + case dm.EDSServiceName != b.EDSServiceName: + return false + case dm.DNSHostname != b.DNSHostname: + return false + } + return true +} + +func equalStringP(a, b *string) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return *a == *b +} + +func equalUint32P(a, b *uint32) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return *a == *b +} + +// LBConfig is the config for cluster resolver balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + // DiscoveryMechanisms is an ordered list of discovery mechanisms. + // + // Must have at least one element. Results from each discovery mechanism are + // concatenated together in successive priorities. + DiscoveryMechanisms []DiscoveryMechanism `json:"discoveryMechanisms,omitempty"` + + // XDSLBPolicy specifies the policy for locality picking and endpoint picking. + // + // Note that it's not normal balancing policy, and it can only be either + // ROUND_ROBIN or RING_HASH. + // + // For ROUND_ROBIN, the policy name will be "ROUND_ROBIN", and the config + // will be empty. This sets the locality-picking policy to weighted_target + // and the endpoint-picking policy to round_robin. + // + // For RING_HASH, the policy name will be "RING_HASH", and the config will + // be lb config for the ring_hash_experimental LB Policy. ring_hash policy + // is responsible for both locality picking and endpoint picking. + XDSLBPolicy *internalserviceconfig.BalancerConfig `json:"xdsLbPolicy,omitempty"` +} + +const ( + rrName = roundrobin.Name + rhName = ringhash.Name +) + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + if lbp := cfg.XDSLBPolicy; lbp != nil && !strings.EqualFold(lbp.Name, rrName) && !strings.EqualFold(lbp.Name, rhName) { + return nil, fmt.Errorf("unsupported child policy with name %q, not one of {%q,%q}", lbp.Name, rrName, rhName) + } + return &cfg, nil +} diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go new file mode 100644 index 0000000000..c135e95152 --- /dev/null +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -0,0 +1,363 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "encoding/json" + "fmt" + "sort" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" + "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/balancer/weightedtarget" + "google.golang.org/grpc/resolver" +) + +const million = 1000000 + +// priorityConfig is config for one priority. For example, if there an EDS and a +// DNS, the priority list will be [priorityConfig{EDS}, priorityConfig{DNS}]. +// +// Each priorityConfig corresponds to one discovery mechanism from the LBConfig +// generated by the CDS balancer. The CDS balancer resolves the cluster name to +// an ordered list of discovery mechanisms (if the top cluster is an aggregated +// cluster), one for each underlying cluster. +type priorityConfig struct { + mechanism DiscoveryMechanism + // edsResp is set only if type is EDS. + edsResp resource.EndpointsUpdate + // addresses is set only if type is DNS. + addresses []string +} + +// buildPriorityConfigJSON builds balancer config for the passed in +// priorities. +// +// The built tree of balancers (see test for the output struct). +// +// If xds lb policy is ROUND_ROBIN, the children will be weighted_target for +// locality picking, and round_robin for endpoint picking. +// +// ┌────────┐ +// │priority│ +// └┬──────┬┘ +// │ │ +// ┌───────────▼┐ ┌▼───────────┐ +// │cluster_impl│ │cluster_impl│ +// └─┬──────────┘ └──────────┬─┘ +// │ │ +// ┌──────────────▼─┐ ┌─▼──────────────┐ +// │locality_picking│ │locality_picking│ +// └┬──────────────┬┘ └┬──────────────┬┘ +// │ │ │ │ +// ┌─▼─┐ ┌─▼─┐ ┌─▼─┐ ┌─▼─┐ +// │LRS│ │LRS│ │LRS│ │LRS│ +// └─┬─┘ └─┬─┘ └─┬─┘ └─┬─┘ +// │ │ │ │ +// ┌──────────▼─────┐ ┌─────▼──────────┐ ┌──────────▼─────┐ ┌─────▼──────────┐ +// │endpoint_picking│ │endpoint_picking│ │endpoint_picking│ │endpoint_picking│ +// └────────────────┘ └────────────────┘ └────────────────┘ └────────────────┘ +// +// If xds lb policy is RING_HASH, the children will be just a ring_hash policy. +// The endpoints from all localities will be flattened to one addresses list, +// and the ring_hash policy will pick endpoints from it. +// +// ┌────────┐ +// │priority│ +// └┬──────┬┘ +// │ │ +// ┌──────────▼─┐ ┌─▼──────────┐ +// │cluster_impl│ │cluster_impl│ +// └──────┬─────┘ └─────┬──────┘ +// │ │ +// ┌──────▼─────┐ ┌─────▼──────┐ +// │ ring_hash │ │ ring_hash │ +// └────────────┘ └────────────┘ +// +// If endpointPickingPolicy is nil, roundrobin will be used. +// +// Custom locality picking policy isn't support, and weighted_target is always +// used. +func buildPriorityConfigJSON(priorities []priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]byte, []resolver.Address, error) { + pc, addrs, err := buildPriorityConfig(priorities, xdsLBPolicy) + if err != nil { + return nil, nil, fmt.Errorf("failed to build priority config: %v", err) + } + ret, err := json.Marshal(pc) + if err != nil { + return nil, nil, fmt.Errorf("failed to marshal built priority config struct into json: %v", err) + } + return ret, addrs, nil +} + +func buildPriorityConfig(priorities []priorityConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*priority.LBConfig, []resolver.Address, error) { + var ( + retConfig = &priority.LBConfig{Children: make(map[string]*priority.Child)} + retAddrs []resolver.Address + ) + for i, p := range priorities { + switch p.mechanism.Type { + case DiscoveryMechanismTypeEDS: + names, configs, addrs, err := buildClusterImplConfigForEDS(i, p.edsResp, p.mechanism, xdsLBPolicy) + if err != nil { + return nil, nil, err + } + retConfig.Priorities = append(retConfig.Priorities, names...) + for n, c := range configs { + retConfig.Children[n] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: c}, + // Ignore all re-resolution from EDS children. + IgnoreReresolutionRequests: true, + } + } + retAddrs = append(retAddrs, addrs...) + case DiscoveryMechanismTypeLogicalDNS: + name, config, addrs := buildClusterImplConfigForDNS(i, p.addresses) + retConfig.Priorities = append(retConfig.Priorities, name) + retConfig.Children[name] = &priority.Child{ + Config: &internalserviceconfig.BalancerConfig{Name: clusterimpl.Name, Config: config}, + // Not ignore re-resolution from DNS children, they will trigger + // DNS to re-resolve. + IgnoreReresolutionRequests: false, + } + retAddrs = append(retAddrs, addrs...) + } + } + return retConfig, retAddrs, nil +} + +func buildClusterImplConfigForDNS(parentPriority int, addrStrs []string) (string, *clusterimpl.LBConfig, []resolver.Address) { + // Endpoint picking policy for DNS is hardcoded to pick_first. + const childPolicy = "pick_first" + retAddrs := make([]resolver.Address, 0, len(addrStrs)) + pName := fmt.Sprintf("priority-%v", parentPriority) + for _, addrStr := range addrStrs { + retAddrs = append(retAddrs, hierarchy.Set(resolver.Address{Addr: addrStr}, []string{pName})) + } + return pName, &clusterimpl.LBConfig{ChildPolicy: &internalserviceconfig.BalancerConfig{Name: childPolicy}}, retAddrs +} + +// buildClusterImplConfigForEDS returns a list of cluster_impl configs, one for +// each priority, sorted by priority, and the addresses for each priority (with +// hierarchy attributes set). +// +// For example, if there are two priorities, the returned values will be +// - ["p0", "p1"] +// - map{"p0":p0_config, "p1":p1_config} +// - [p0_address_0, p0_address_1, p1_address_0, p1_address_1] +// - p0 addresses' hierarchy attributes are set to p0 +func buildClusterImplConfigForEDS(parentPriority int, edsResp resource.EndpointsUpdate, mechanism DiscoveryMechanism, xdsLBPolicy *internalserviceconfig.BalancerConfig) ([]string, map[string]*clusterimpl.LBConfig, []resolver.Address, error) { + drops := make([]clusterimpl.DropConfig, 0, len(edsResp.Drops)) + for _, d := range edsResp.Drops { + drops = append(drops, clusterimpl.DropConfig{ + Category: d.Category, + RequestsPerMillion: d.Numerator * million / d.Denominator, + }) + } + + priorityChildNames, priorities := groupLocalitiesByPriority(edsResp.Localities) + retNames := make([]string, 0, len(priorityChildNames)) + retAddrs := make([]resolver.Address, 0, len(priorityChildNames)) + retConfigs := make(map[string]*clusterimpl.LBConfig, len(priorityChildNames)) + for _, priorityName := range priorityChildNames { + priorityLocalities := priorities[priorityName] + // Prepend parent priority to the priority names, to avoid duplicates. + pName := fmt.Sprintf("priority-%v-%v", parentPriority, priorityName) + retNames = append(retNames, pName) + cfg, addrs, err := priorityLocalitiesToClusterImpl(priorityLocalities, pName, mechanism, drops, xdsLBPolicy) + if err != nil { + return nil, nil, nil, err + } + retConfigs[pName] = cfg + retAddrs = append(retAddrs, addrs...) + } + return retNames, retConfigs, retAddrs, nil +} + +// groupLocalitiesByPriority returns the localities grouped by priority. +// +// It also returns a list of strings where each string represents a priority, +// and the list is sorted from higher priority to lower priority. +// +// For example, for L0-p0, L1-p0, L2-p1, results will be +// - ["p0", "p1"] +// - map{"p0":[L0, L1], "p1":[L2]} +func groupLocalitiesByPriority(localities []resource.Locality) ([]string, map[string][]resource.Locality) { + var priorityIntSlice []int + priorities := make(map[string][]resource.Locality) + for _, locality := range localities { + if locality.Weight == 0 { + continue + } + priorityName := fmt.Sprintf("%v", locality.Priority) + priorities[priorityName] = append(priorities[priorityName], locality) + priorityIntSlice = append(priorityIntSlice, int(locality.Priority)) + } + // Sort the priorities based on the int value, deduplicate, and then turn + // the sorted list into a string list. This will be child names, in priority + // order. + sort.Ints(priorityIntSlice) + priorityIntSliceDeduped := dedupSortedIntSlice(priorityIntSlice) + priorityNameSlice := make([]string, 0, len(priorityIntSliceDeduped)) + for _, p := range priorityIntSliceDeduped { + priorityNameSlice = append(priorityNameSlice, fmt.Sprintf("%v", p)) + } + return priorityNameSlice, priorities +} + +func dedupSortedIntSlice(a []int) []int { + if len(a) == 0 { + return a + } + i, j := 0, 1 + for ; j < len(a); j++ { + if a[i] == a[j] { + continue + } + i++ + if i != j { + a[i] = a[j] + } + } + return a[:i+1] +} + +// rrBalancerConfig is a const roundrobin config, used as child of +// weighted-roundrobin. To avoid allocating memory everytime. +var rrBalancerConfig = &internalserviceconfig.BalancerConfig{Name: roundrobin.Name} + +// priorityLocalitiesToClusterImpl takes a list of localities (with the same +// priority), and generates a cluster impl policy config, and a list of +// addresses. +func priorityLocalitiesToClusterImpl(localities []resource.Locality, priorityName string, mechanism DiscoveryMechanism, drops []clusterimpl.DropConfig, xdsLBPolicy *internalserviceconfig.BalancerConfig) (*clusterimpl.LBConfig, []resolver.Address, error) { + clusterImplCfg := &clusterimpl.LBConfig{ + Cluster: mechanism.Cluster, + EDSServiceName: mechanism.EDSServiceName, + LoadReportingServerName: mechanism.LoadReportingServerName, + MaxConcurrentRequests: mechanism.MaxConcurrentRequests, + DropCategories: drops, + // ChildPolicy is not set. Will be set based on xdsLBPolicy + } + + if xdsLBPolicy == nil || xdsLBPolicy.Name == rrName { + // If lb policy is ROUND_ROBIN: + // - locality-picking policy is weighted_target + // - endpoint-picking policy is round_robin + logger.Infof("xds lb policy is %q, building config with weighted_target + round_robin", rrName) + // Child of weighted_target is hardcoded to round_robin. + wtConfig, addrs := localitiesToWeightedTarget(localities, priorityName, rrBalancerConfig) + clusterImplCfg.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: weightedtarget.Name, Config: wtConfig} + return clusterImplCfg, addrs, nil + } + + if xdsLBPolicy.Name == rhName { + // If lb policy is RIHG_HASH, will build one ring_hash policy as child. + // The endpoints from all localities will be flattened to one addresses + // list, and the ring_hash policy will pick endpoints from it. + logger.Infof("xds lb policy is %q, building config with ring_hash", rhName) + addrs := localitiesToRingHash(localities, priorityName) + // Set child to ring_hash, note that the ring_hash config is from + // xdsLBPolicy. + clusterImplCfg.ChildPolicy = &internalserviceconfig.BalancerConfig{Name: ringhash.Name, Config: xdsLBPolicy.Config} + return clusterImplCfg, addrs, nil + } + + return nil, nil, fmt.Errorf("unsupported xds LB policy %q, not one of {%q,%q}", xdsLBPolicy.Name, rrName, rhName) +} + +// localitiesToRingHash takes a list of localities (with the same priority), and +// generates a list of addresses. +// +// The addresses have path hierarchy set to [priority-name], so priority knows +// which child policy they are for. +func localitiesToRingHash(localities []resource.Locality, priorityName string) []resolver.Address { + var addrs []resolver.Address + for _, locality := range localities { + var lw uint32 = 1 + if locality.Weight != 0 { + lw = locality.Weight + } + localityStr, err := locality.ID.ToString() + if err != nil { + localityStr = fmt.Sprintf("%+v", locality.ID) + } + for _, endpoint := range locality.Endpoints { + // Filter out all "unhealthy" endpoints (unknown and healthy are + // both considered to be healthy: + // https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus). + if endpoint.HealthStatus != resource.EndpointHealthStatusHealthy && endpoint.HealthStatus != resource.EndpointHealthStatusUnknown { + continue + } + + var ew uint32 = 1 + if endpoint.Weight != 0 { + ew = endpoint.Weight + } + + // The weight of each endpoint is locality_weight * endpoint_weight. + ai := weightedroundrobin.AddrInfo{Weight: lw * ew} + addr := weightedroundrobin.SetAddrInfo(resolver.Address{Addr: endpoint.Address}, ai) + addr = hierarchy.Set(addr, []string{priorityName, localityStr}) + addr = resource.SetLocalityID(addr, locality.ID) + addrs = append(addrs, addr) + } + } + return addrs +} + +// localitiesToWeightedTarget takes a list of localities (with the same +// priority), and generates a weighted target config, and list of addresses. +// +// The addresses have path hierarchy set to [priority-name, locality-name], so +// priority and weighted target know which child policy they are for. +func localitiesToWeightedTarget(localities []resource.Locality, priorityName string, childPolicy *internalserviceconfig.BalancerConfig) (*TargetLBConfig, []resolver.Address) { + weightedTargets := make(map[string]Target) + var addrs []resolver.Address + for _, locality := range localities { + localityStr, err := locality.ID.ToString() + if err != nil { + localityStr = fmt.Sprintf("%+v", locality.ID) + } + weightedTargets[localityStr] = Target{Weight: locality.Weight, ChildPolicy: childPolicy} + for _, endpoint := range locality.Endpoints { + // Filter out all "unhealthy" endpoints (unknown and healthy are + // both considered to be healthy: + // https://www.envoyproxy.io/docs/envoy/latest/api-v2/api/v2/core/health_check.proto#envoy-api-enum-core-healthstatus). + if endpoint.HealthStatus != resource.EndpointHealthStatusHealthy && endpoint.HealthStatus != resource.EndpointHealthStatusUnknown { + continue + } + + addr := resolver.Address{Addr: endpoint.Address} + if childPolicy.Name == weightedroundrobin.Name && endpoint.Weight != 0 { + ai := weightedroundrobin.AddrInfo{Weight: endpoint.Weight} + addr = weightedroundrobin.SetAddrInfo(addr, ai) + } + addr = hierarchy.Set(addr, []string{priorityName, localityStr}) + addr = resource.SetLocalityID(addr, locality.ID) + addrs = append(addrs, addr) + } + } + return &TargetLBConfig{Targets: weightedTargets}, addrs +} diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go new file mode 100644 index 0000000000..b30739bb9c --- /dev/null +++ b/xds/balancer/clusterresolver/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[xds-cluster-resolver-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clusterResolverBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go new file mode 100644 index 0000000000..d4dbc321da --- /dev/null +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -0,0 +1,248 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// resourceUpdate is a combined update from all the resources, in the order of +// priority. For example, it can be {EDS, EDS, DNS}. +type resourceUpdate struct { + priorities []priorityConfig + err error +} + +type discoveryMechanism interface { + lastUpdate() (interface{}, bool) + resolveNow() + stop() +} + +// discoveryMechanismKey is {type+resource_name}, it's used as the map key, so +// that the same resource resolver can be reused (e.g. when there are two +// mechanisms, both for the same EDS resource, but has different circuit +// breaking config. +type discoveryMechanismKey struct { + typ DiscoveryMechanismType + name string +} + +// resolverMechanismTuple is needed to keep the resolver and the discovery +// mechanism together, because resolvers can be shared. And we need the +// mechanism for fields like circuit breaking, LRS etc when generating the +// balancer config. +type resolverMechanismTuple struct { + dm DiscoveryMechanism + dmKey discoveryMechanismKey + r discoveryMechanism +} + +type resourceResolver struct { + parent *clusterResolverBalancer + updateChannel chan *resourceUpdate + + // mu protects the slice and map, and content of the resolvers in the slice. + mu sync.Mutex + mechanisms []DiscoveryMechanism + children []resolverMechanismTuple + childrenMap map[discoveryMechanismKey]discoveryMechanism +} + +func newResourceResolver(parent *clusterResolverBalancer) *resourceResolver { + return &resourceResolver{ + parent: parent, + updateChannel: make(chan *resourceUpdate, 1), + childrenMap: make(map[discoveryMechanismKey]discoveryMechanism), + } +} + +func equalDiscoveryMechanisms(a, b []DiscoveryMechanism) bool { + if len(a) != len(b) { + return false + } + for i, aa := range a { + bb := b[i] + if !aa.Equal(bb) { + return false + } + } + return true +} + +func (rr *resourceResolver) updateMechanisms(mechanisms []DiscoveryMechanism) { + rr.mu.Lock() + defer rr.mu.Unlock() + if equalDiscoveryMechanisms(rr.mechanisms, mechanisms) { + return + } + rr.mechanisms = mechanisms + rr.children = make([]resolverMechanismTuple, len(mechanisms)) + newDMs := make(map[discoveryMechanismKey]bool) + + // Start one watch for each new discover mechanism {type+resource_name}. + for i, dm := range mechanisms { + switch dm.Type { + case DiscoveryMechanismTypeEDS: + // If EDSServiceName is not set, use the cluster name as EDS service + // name to watch. + nameToWatch := dm.EDSServiceName + if nameToWatch == "" { + nameToWatch = dm.Cluster + } + dmKey := discoveryMechanismKey{typ: dm.Type, name: nameToWatch} + newDMs[dmKey] = true + + r := rr.childrenMap[dmKey] + if r == nil { + r = newEDSResolver(nameToWatch, rr.parent.xdsClient, rr) + rr.childrenMap[dmKey] = r + } + rr.children[i] = resolverMechanismTuple{dm: dm, dmKey: dmKey, r: r} + case DiscoveryMechanismTypeLogicalDNS: + // Name to resolve in DNS is the hostname, not the ClientConn + // target. + dmKey := discoveryMechanismKey{typ: dm.Type, name: dm.DNSHostname} + newDMs[dmKey] = true + + r := rr.childrenMap[dmKey] + if r == nil { + r = newDNSResolver(dm.DNSHostname, rr) + rr.childrenMap[dmKey] = r + } + rr.children[i] = resolverMechanismTuple{dm: dm, dmKey: dmKey, r: r} + } + } + // Stop the resources that were removed. + for dm, r := range rr.childrenMap { + if !newDMs[dm] { + delete(rr.childrenMap, dm) + r.stop() + } + } + // Regenerate even if there's no change in discovery mechanism, in case + // priority order changed. + rr.generate() +} + +// resolveNow is typically called to trigger re-resolve of DNS. The EDS +// resolveNow() is a noop. +func (rr *resourceResolver) resolveNow() { + rr.mu.Lock() + defer rr.mu.Unlock() + for _, r := range rr.childrenMap { + r.resolveNow() + } +} + +func (rr *resourceResolver) stop() { + rr.mu.Lock() + defer rr.mu.Unlock() + for dm, r := range rr.childrenMap { + delete(rr.childrenMap, dm) + r.stop() + } + rr.mechanisms = nil + rr.children = nil +} + +// generate collects all the updates from all the resolvers, and push the +// combined result into the update channel. It only pushes the update when all +// the child resolvers have received at least one update, otherwise it will +// wait. +// +// caller must hold rr.mu. +func (rr *resourceResolver) generate() { + var ret []priorityConfig + for _, rDM := range rr.children { + r, ok := rr.childrenMap[rDM.dmKey] + if !ok { + rr.parent.logger.Infof("resolver for %+v not found, should never happen", rDM.dmKey) + continue + } + + u, ok := r.lastUpdate() + if !ok { + // Don't send updates to parent until all resolvers have update to + // send. + return + } + switch uu := u.(type) { + case resource.EndpointsUpdate: + ret = append(ret, priorityConfig{mechanism: rDM.dm, edsResp: uu}) + case []string: + ret = append(ret, priorityConfig{mechanism: rDM.dm, addresses: uu}) + } + } + select { + case <-rr.updateChannel: + default: + } + rr.updateChannel <- &resourceUpdate{priorities: ret} +} + +type edsDiscoveryMechanism struct { + cancel func() + + update resource.EndpointsUpdate + updateReceived bool +} + +func (er *edsDiscoveryMechanism) lastUpdate() (interface{}, bool) { + if !er.updateReceived { + return nil, false + } + return er.update, true +} + +func (er *edsDiscoveryMechanism) resolveNow() { +} + +func (er *edsDiscoveryMechanism) stop() { + er.cancel() +} + +// newEDSResolver starts the EDS watch on the given xds client. +func newEDSResolver(nameToWatch string, xdsc client.XDSClient, topLevelResolver *resourceResolver) *edsDiscoveryMechanism { + ret := &edsDiscoveryMechanism{} + topLevelResolver.parent.logger.Infof("EDS watch started on %v", nameToWatch) + cancel := xdsc.WatchEndpoints(nameToWatch, func(update resource.EndpointsUpdate, err error) { + topLevelResolver.mu.Lock() + defer topLevelResolver.mu.Unlock() + if err != nil { + select { + case <-topLevelResolver.updateChannel: + default: + } + topLevelResolver.updateChannel <- &resourceUpdate{err: err} + return + } + ret.update = update + ret.updateReceived = true + topLevelResolver.generate() + }) + ret.cancel = func() { + topLevelResolver.parent.logger.Infof("EDS watch canceled on %v", nameToWatch) + cancel() + } + return ret +} diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go new file mode 100644 index 0000000000..7a639f51a5 --- /dev/null +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + "fmt" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +var ( + newDNS = func(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + // The dns resolver is registered by the grpc package. So, this call to + // resolver.Get() is never expected to return nil. + return resolver.Get("dns").Build(target, cc, opts) + } +) + +// dnsDiscoveryMechanism watches updates for the given DNS hostname. +// +// It implements resolver.ClientConn interface to work with the DNS resolver. +type dnsDiscoveryMechanism struct { + target string + topLevelResolver *resourceResolver + r resolver.Resolver + + addrs []string + updateReceived bool +} + +func newDNSResolver(target string, topLevelResolver *resourceResolver) *dnsDiscoveryMechanism { + ret := &dnsDiscoveryMechanism{ + target: target, + topLevelResolver: topLevelResolver, + } + r, err := newDNS(resolver.Target{Scheme: "dns", Endpoint: target}, ret, resolver.BuildOptions{}) + if err != nil { + select { + case <-topLevelResolver.updateChannel: + default: + } + topLevelResolver.updateChannel <- &resourceUpdate{err: err} + } + ret.r = r + return ret +} + +func (dr *dnsDiscoveryMechanism) lastUpdate() (interface{}, bool) { + if !dr.updateReceived { + return nil, false + } + return dr.addrs, true +} + +func (dr *dnsDiscoveryMechanism) resolveNow() { + dr.r.ResolveNow(resolver.ResolveNowOptions{}) +} + +func (dr *dnsDiscoveryMechanism) stop() { + dr.r.Close() +} + +// dnsDiscoveryMechanism needs to implement resolver.ClientConn interface to receive +// updates from the real DNS resolver. + +func (dr *dnsDiscoveryMechanism) UpdateState(state resolver.State) error { + dr.topLevelResolver.mu.Lock() + defer dr.topLevelResolver.mu.Unlock() + addrs := make([]string, len(state.Addresses)) + for i, a := range state.Addresses { + addrs[i] = a.Addr + } + dr.addrs = addrs + dr.updateReceived = true + dr.topLevelResolver.generate() + return nil +} + +func (dr *dnsDiscoveryMechanism) ReportError(err error) { + select { + case <-dr.topLevelResolver.updateChannel: + default: + } + dr.topLevelResolver.updateChannel <- &resourceUpdate{err: err} +} + +func (dr *dnsDiscoveryMechanism) NewAddress(addresses []resolver.Address) { + dr.UpdateState(resolver.State{Addresses: addresses}) +} + +func (dr *dnsDiscoveryMechanism) NewServiceConfig(string) { + // This method is deprecated, and service config isn't supported. +} + +func (dr *dnsDiscoveryMechanism) ParseServiceConfig(string) *serviceconfig.ParseResult { + return &serviceconfig.ParseResult{Err: fmt.Errorf("service config not supported")} +} diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go new file mode 100644 index 0000000000..219a857bd3 --- /dev/null +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package clusterresolver + +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "encoding/json" + "google.golang.org/grpc/serviceconfig" +) + +// Target represents one target with the weight and the child policy. +type Target struct { + // Weight is the weight of the child policy. + Weight uint32 `json:"weight,omitempty"` + // ChildPolicy is the child policy and it's config. + ChildPolicy *internalserviceconfig.BalancerConfig `json:"childPolicy,omitempty"` +} + +// LBConfig is the balancer config for weighted_target. +type TargetLBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + Targets map[string]Target `json:"targets,omitempty"` +} + +func parseTargetConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + return &cfg, nil +} diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go new file mode 100644 index 0000000000..8133d6d4af --- /dev/null +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -0,0 +1,120 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package loadstore contains the loadStoreWrapper shared by the balancers. +package loadstore + +import ( + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/client/load" +) + +// NewWrapper creates a Wrapper. +func NewWrapper() *Wrapper { + return &Wrapper{} +} + +// Wrapper wraps a load store with cluster and edsService. +// +// It's store and cluster/edsService can be updated separately. And it will +// update its internal perCluster store so that new stats will be added to the +// correct perCluster. +// +// Note that this struct is a temporary walkaround before we implement graceful +// switch for EDS. Any update to the clusterName and serviceName is too early, +// the perfect timing is when the picker is updated with the new connection. +// This early update could cause picks for the old SubConn being reported to the +// new services. +// +// When the graceful switch in EDS is done, there should be no need for this +// struct. The policies that record/report load shouldn't need to handle update +// of lrsServerName/cluster/edsService. Its parent should do a graceful switch +// of the whole tree when one of that changes. +type Wrapper struct { + mu sync.RWMutex + cluster string + edsService string + // store and perCluster are initialized as nil. They are only set by the + // balancer when LRS is enabled. Before that, all functions to record loads + // are no-op. + store *load.Store + perCluster load.PerClusterReporter +} + +// UpdateClusterAndService updates the cluster name and eds service for this +// wrapper. If any one of them is changed from before, the perCluster store in +// this wrapper will also be updated. +func (lsw *Wrapper) UpdateClusterAndService(cluster, edsService string) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if cluster == lsw.cluster && edsService == lsw.edsService { + return + } + lsw.cluster = cluster + lsw.edsService = edsService + lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService) +} + +// UpdateLoadStore updates the load store for this wrapper. If it is changed +// from before, the perCluster store in this wrapper will also be updated. +func (lsw *Wrapper) UpdateLoadStore(store *load.Store) { + lsw.mu.Lock() + defer lsw.mu.Unlock() + if store == lsw.store { + return + } + lsw.store = store + lsw.perCluster = lsw.store.PerCluster(lsw.cluster, lsw.edsService) +} + +// CallStarted records a call started in the store. +func (lsw *Wrapper) CallStarted(locality string) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallStarted(locality) + } +} + +// CallFinished records a call finished in the store. +func (lsw *Wrapper) CallFinished(locality string, err error) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallFinished(locality, err) + } +} + +// CallServerLoad records the server load in the store. +func (lsw *Wrapper) CallServerLoad(locality, name string, val float64) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallServerLoad(locality, name, val) + } +} + +// CallDropped records a call dropped in the store. +func (lsw *Wrapper) CallDropped(category string) { + lsw.mu.RLock() + defer lsw.mu.RUnlock() + if lsw.perCluster != nil { + lsw.perCluster.CallDropped(category) + } +} diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go new file mode 100644 index 0000000000..bd32992986 --- /dev/null +++ b/xds/balancer/orca/orca.go @@ -0,0 +1,84 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package orca implements Open Request Cost Aggregation. +package orca + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancerload" + orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" +) + +const mdKey = "X-Endpoint-Load-Metrics-Bin" + +var logger = grpclog.Component("xds") + +// toBytes converts a orca load report into bytes. +func toBytes(r *orcapb.OrcaLoadReport) []byte { + if r == nil { + return nil + } + + b, err := proto.Marshal(r) + if err != nil { + logger.Warningf("orca: failed to marshal load report: %v", err) + return nil + } + return b +} + +// ToMetadata converts a orca load report into grpc metadata. +func ToMetadata(r *orcapb.OrcaLoadReport) metadata.MD { + b := toBytes(r) + if b == nil { + return nil + } + return metadata.Pairs(mdKey, string(b)) +} + +// fromBytes reads load report bytes and converts it to orca. +func fromBytes(b []byte) *orcapb.OrcaLoadReport { + ret := new(orcapb.OrcaLoadReport) + if err := proto.Unmarshal(b, ret); err != nil { + logger.Warningf("orca: failed to unmarshal load report: %v", err) + return nil + } + return ret +} + +// FromMetadata reads load report from metadata and converts it to orca. +// +// It returns nil if report is not found in metadata. +func FromMetadata(md metadata.MD) *orcapb.OrcaLoadReport { + vs := md.Get(mdKey) + if len(vs) == 0 { + return nil + } + return fromBytes([]byte(vs[0])) +} + +type loadParser struct{} + +func (*loadParser) Parse(md metadata.MD) interface{} { + return FromMetadata(md) +} + +func init() { + balancerload.SetParser(&loadParser{}) +} diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go new file mode 100644 index 0000000000..f2111fd344 --- /dev/null +++ b/xds/balancer/priority/balancer.go @@ -0,0 +1,253 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package priority implements the priority balancer. +// +// This balancer will be kept in internal until we use it in the xds balancers, +// and are confident its functionalities are stable. It will then be exported +// for more users. +package priority + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the priority balancer. +const Name = "priority_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &priorityBalancer{ + cc: cc, + done: grpcsync.NewEvent(), + childToPriority: make(map[string]int), + children: make(map[string]*childBalancer), + childBalancerStateUpdate: buffer.NewUnbounded(), + } + + b.logger = prefixLogger(b) + b.bg = balancergroup.New(cc, bOpts, b, b.logger) + b.bg.Start() + go b.run() + b.logger.Infof("Created") + return b +} + +func (b bb) ParseConfig(s json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(s) +} + +func (bb) Name() string { + return Name +} + +// timerWrapper wraps a timer with a boolean. So that when a race happens +// between AfterFunc and Stop, the func is guaranteed to not execute. +type timerWrapper struct { + stopped bool + timer *time.Timer +} + +type priorityBalancer struct { + logger *grpclog.PrefixLogger + cc balancer.ClientConn + bg *balancergroup.BalancerGroup + done *grpcsync.Event + childBalancerStateUpdate *buffer.Unbounded + + mu sync.Mutex + childInUse string + // priority of the child that's current in use. Int starting from 0, and 0 + // is the higher priority. + priorityInUse int + // priorities is a list of child names from higher to lower priority. + priorities []string + // childToPriority is a map from the child name to it's priority. Priority + // is an int start from 0, and 0 is the higher priority. + childToPriority map[string]int + // children is a map from child name to sub-balancers. + children map[string]*childBalancer + // The timer to give a priority some time to connect. And if the priority + // doesn't go into Ready/Failure, the next priority will be started. + // + // One timer is enough because there can be at most one priority in init + // state. + priorityInitTimer *timerWrapper +} + +func (b *priorityBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + addressesSplit := hierarchy.Group(s.ResolverState.Addresses) + + b.mu.Lock() + defer b.mu.Unlock() + // Create and remove children, since we know all children from the config + // are used by some priority. + for name, newSubConfig := range newConfig.Children { + bb := balancer.Get(newSubConfig.Config.Name) + if bb == nil { + b.logger.Errorf("balancer name %v from config is not registered", newSubConfig.Config.Name) + continue + } + + currentChild, ok := b.children[name] + if !ok { + // This is a new child, add it to the children list. But note that + // the balancer isn't built, because this child can be a low + // priority. If necessary, it will be built when syncing priorities. + cb := newChildBalancer(name, b, bb) + cb.updateConfig(newSubConfig, resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + b.children[name] = cb + continue + } + + // This is not a new child. But the config/addresses could change. + + // The balancing policy name is changed, close the old child. But don't + // rebuild, rebuild will happen when syncing priorities. + if currentChild.bb.Name() != bb.Name() { + currentChild.stop() + currentChild.updateBuilder(bb) + } + + // Update config and address, but note that this doesn't send the + // updates to child balancer (the child balancer might not be built, if + // it's a low priority). + currentChild.updateConfig(newSubConfig, resolver.State{ + Addresses: addressesSplit[name], + ServiceConfig: s.ResolverState.ServiceConfig, + Attributes: s.ResolverState.Attributes, + }) + } + + // Remove child from children if it's not in new config. + for name, oldChild := range b.children { + if _, ok := newConfig.Children[name]; !ok { + oldChild.stop() + } + } + + // Update priorities and handle priority changes. + b.priorities = newConfig.Priorities + b.childToPriority = make(map[string]int, len(newConfig.Priorities)) + for pi, pName := range newConfig.Priorities { + b.childToPriority[pName] = pi + } + // Sync the states of all children to the new updated priorities. This + // include starting/stopping child balancers when necessary. + b.syncPriority() + + return nil +} + +func (b *priorityBalancer) ResolverError(err error) { + b.bg.ResolverError(err) +} + +func (b *priorityBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b.bg.UpdateSubConnState(sc, state) +} + +func (b *priorityBalancer) Close() { + b.bg.Close() + + b.mu.Lock() + defer b.mu.Unlock() + b.done.Fire() + // Clear states of the current child in use, so if there's a race in picker + // update, it will be dropped. + b.childInUse = "" + b.stopPriorityInitTimer() +} + +func (b *priorityBalancer) ExitIdle() { + b.bg.ExitIdle() +} + +// stopPriorityInitTimer stops the priorityInitTimer if it's not nil, and set it +// to nil. +// +// Caller must hold b.mu. +func (b *priorityBalancer) stopPriorityInitTimer() { + timerW := b.priorityInitTimer + if timerW == nil { + return + } + b.priorityInitTimer = nil + timerW.stopped = true + timerW.timer.Stop() +} + +// UpdateState implements balancergroup.BalancerStateAggregator interface. The +// balancer group sends new connectivity state and picker here. +func (b *priorityBalancer) UpdateState(childName string, state balancer.State) { + b.childBalancerStateUpdate.Put(&childBalancerState{ + name: childName, + s: state, + }) +} + +type childBalancerState struct { + name string + s balancer.State +} + +// run handles child update in a separate goroutine, so if the child sends +// updates inline (when called by parent), it won't cause deadlocks (by trying +// to hold the same mutex). +func (b *priorityBalancer) run() { + for { + select { + case u := <-b.childBalancerStateUpdate.Get(): + b.childBalancerStateUpdate.Load() + s := u.(*childBalancerState) + // Needs to handle state update in a goroutine, because each state + // update needs to start/close child policy, could result in + // deadlock. + b.handleChildStateUpdate(s.name, s.s) + case <-b.done.Done(): + return + } + } +} diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go new file mode 100644 index 0000000000..600705da01 --- /dev/null +++ b/xds/balancer/priority/balancer_child.go @@ -0,0 +1,112 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +type childBalancer struct { + name string + parent *priorityBalancer + bb *ignoreResolveNowBalancerBuilder + + ignoreReresolutionRequests bool + config serviceconfig.LoadBalancingConfig + rState resolver.State + + started bool + state balancer.State +} + +// newChildBalancer creates a child balancer place holder, but doesn't +// build/start the child balancer. +func newChildBalancer(name string, parent *priorityBalancer, bb balancer.Builder) *childBalancer { + return &childBalancer{ + name: name, + parent: parent, + bb: newIgnoreResolveNowBalancerBuilder(bb, false), + started: false, + // Start with the connecting state and picker with re-pick error, so + // that when a priority switch causes this child picked before it's + // balancing policy is created, a re-pick will happen. + state: balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + }, + } +} + +// updateBuilder updates builder for the child, but doesn't build. +func (cb *childBalancer) updateBuilder(bb balancer.Builder) { + cb.bb = newIgnoreResolveNowBalancerBuilder(bb, cb.ignoreReresolutionRequests) +} + +// updateConfig sets childBalancer's config and state, but doesn't send update to +// the child balancer. +func (cb *childBalancer) updateConfig(child *Child, rState resolver.State) { + cb.ignoreReresolutionRequests = child.IgnoreReresolutionRequests + cb.config = child.Config.Config + cb.rState = rState +} + +// start builds the child balancer if it's not already started. +// +// It doesn't do it directly. It asks the balancer group to build it. +func (cb *childBalancer) start() { + if cb.started { + return + } + cb.started = true + cb.parent.bg.Add(cb.name, cb.bb) +} + +// sendUpdate sends the addresses and config to the child balancer. +func (cb *childBalancer) sendUpdate() { + cb.bb.updateIgnoreResolveNow(cb.ignoreReresolutionRequests) + // TODO: return and aggregate the returned error in the parent. + err := cb.parent.bg.UpdateClientConnState(cb.name, balancer.ClientConnState{ + ResolverState: cb.rState, + BalancerConfig: cb.config, + }) + if err != nil { + cb.parent.logger.Warningf("failed to update ClientConn state for child %v: %v", cb.name, err) + } +} + +// stop stops the child balancer and resets the state. +// +// It doesn't do it directly. It asks the balancer group to remove it. +// +// Note that the underlying balancer group could keep the child in a cache. +func (cb *childBalancer) stop() { + if !cb.started { + return + } + cb.parent.bg.Remove(cb.name) + cb.started = false + cb.state = balancer.State{ + ConnectivityState: connectivity.Connecting, + Picker: base.NewErrPicker(balancer.ErrNoSubConnAvailable), + } +} diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go new file mode 100644 index 0000000000..37cd445604 --- /dev/null +++ b/xds/balancer/priority/balancer_priority.go @@ -0,0 +1,362 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "errors" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" +) + +var ( + // ErrAllPrioritiesRemoved is returned by the picker when there's no priority available. + ErrAllPrioritiesRemoved = errors.New("no priority is provided, all priorities are removed") + // DefaultPriorityInitTimeout is the timeout after which if a priority is + // not READY, the next will be started. It's exported to be overridden by + // tests. + DefaultPriorityInitTimeout = 10 * time.Second +) + +// syncPriority handles priority after a config update. It makes sure the +// balancer state (started or not) is in sync with the priorities (even in +// tricky cases where a child is moved from a priority to another). +// +// It's guaranteed that after this function returns: +// - If some child is READY, it is childInUse, and all lower priorities are +// closed. +// - If some child is newly started(in Connecting for the first time), it is +// childInUse, and all lower priorities are closed. +// - Otherwise, the lowest priority is childInUse (none of the children is +// ready, and the overall state is not ready). +// +// Steps: +// - If all priorities were deleted, unset childInUse (to an empty string), and +// set parent ClientConn to TransientFailure +// - Otherwise, Scan all children from p0, and check balancer stats: +// - For any of the following cases: +// - If balancer is not started (not built), this is either a new child +// with high priority, or a new builder for an existing child. +// - If balancer is READY +// - If this is the lowest priority +// - do the following: +// - if this is not the old childInUse, override picker so old picker is no +// longer used. +// - switch to it (because all higher priorities are neither new or Ready) +// - forward the new addresses and config +// +// Caller must hold b.mu. +func (b *priorityBalancer) syncPriority() { + // Everything was removed by the update. + if len(b.priorities) == 0 { + b.childInUse = "" + b.priorityInUse = 0 + // Stop the init timer. This can happen if the only priority is removed + // shortly after it's added. + b.stopPriorityInitTimer() + b.cc.UpdateState(balancer.State{ + ConnectivityState: connectivity.TransientFailure, + Picker: base.NewErrPicker(ErrAllPrioritiesRemoved), + }) + return + } + + for p, name := range b.priorities { + child, ok := b.children[name] + if !ok { + b.logger.Errorf("child with name %q is not found in children", name) + continue + } + + if !child.started || + child.state.ConnectivityState == connectivity.Ready || + child.state.ConnectivityState == connectivity.Idle || + p == len(b.priorities)-1 { + if b.childInUse != "" && b.childInUse != child.name { + // childInUse was set and is different from this child, will + // change childInUse later. We need to update picker here + // immediately so parent stops using the old picker. + b.cc.UpdateState(child.state) + } + b.logger.Infof("switching to (%q, %v) in syncPriority", child.name, p) + b.switchToChild(child, p) + child.sendUpdate() + break + } + } +} + +// Stop priorities [p+1, lowest]. +// +// Caller must hold b.mu. +func (b *priorityBalancer) stopSubBalancersLowerThanPriority(p int) { + for i := p + 1; i < len(b.priorities); i++ { + name := b.priorities[i] + child, ok := b.children[name] + if !ok { + b.logger.Errorf("child with name %q is not found in children", name) + continue + } + child.stop() + } +} + +// switchToChild does the following: +// - stop all child with lower priorities +// - if childInUse is not this child +// - set childInUse to this child +// - stops init timer +// - if this child is not started, start it, and start a init timer +// +// Note that it does NOT send the current child state (picker) to the parent +// ClientConn. The caller needs to send it if necessary. +// +// this can be called when +// 1. first update, start p0 +// 2. an update moves a READY child from a lower priority to higher +// 2. a different builder is updated for this child +// 3. a high priority goes Failure, start next +// 4. a high priority init timeout, start next +// +// Caller must hold b.mu. +func (b *priorityBalancer) switchToChild(child *childBalancer, priority int) { + // Stop lower priorities even if childInUse is same as this child. It's + // possible this child was moved from a priority to another. + b.stopSubBalancersLowerThanPriority(priority) + + // If this child is already in use, do nothing. + // + // This can happen: + // - all priorities are not READY, an config update always triggers switch + // to the lowest. In this case, the lowest child could still be connecting, + // so we don't stop the init timer. + // - a high priority is READY, an config update always triggers switch to + // it. + if b.childInUse == child.name && child.started { + return + } + b.childInUse = child.name + b.priorityInUse = priority + + // Init timer is always for childInUse. Since we are switching to a + // different child, we will stop the init timer no matter what. If this + // child is not started, we will start the init timer later. + b.stopPriorityInitTimer() + + if !child.started { + child.start() + // Need this local variable to capture timerW in the AfterFunc closure + // to check the stopped boolean. + timerW := &timerWrapper{} + b.priorityInitTimer = timerW + timerW.timer = time.AfterFunc(DefaultPriorityInitTimeout, func() { + b.mu.Lock() + defer b.mu.Unlock() + if timerW.stopped { + return + } + b.priorityInitTimer = nil + // Switch to the next priority if there's any. + if pNext := priority + 1; pNext < len(b.priorities) { + nameNext := b.priorities[pNext] + if childNext, ok := b.children[nameNext]; ok { + b.switchToChild(childNext, pNext) + childNext.sendUpdate() + } + } + }) + } +} + +// handleChildStateUpdate start/close priorities based on the connectivity +// state. +func (b *priorityBalancer) handleChildStateUpdate(childName string, s balancer.State) { + b.mu.Lock() + defer b.mu.Unlock() + if b.done.HasFired() { + return + } + + priority, ok := b.childToPriority[childName] + if !ok { + b.logger.Errorf("priority: received picker update with unknown child %v", childName) + return + } + + if b.childInUse == "" { + b.logger.Errorf("priority: no child is in use when picker update is received") + return + } + + // priorityInUse is higher than this priority. + if b.priorityInUse < priority { + // Lower priorities should all be closed, this is an unexpected update. + // Can happen if the child policy sends an update after we tell it to + // close. + b.logger.Warningf("priority: received picker update from priority %v, lower than priority in use %v", priority, b.priorityInUse) + return + } + + // Update state in child. The updated picker will be sent to parent later if + // necessary. + child, ok := b.children[childName] + if !ok { + b.logger.Errorf("priority: child balancer not found for child %v, priority %v", childName, priority) + return + } + oldState := child.state.ConnectivityState + child.state = s + + switch s.ConnectivityState { + case connectivity.Ready, connectivity.Idle: + // Note that idle is also handled as if it's Ready. It will close the + // lower priorities (which will be kept in a cache, not deleted), and + // new picks will use the Idle picker. + b.handlePriorityWithNewStateReady(child, priority) + case connectivity.TransientFailure: + b.handlePriorityWithNewStateTransientFailure(child, priority) + case connectivity.Connecting: + b.handlePriorityWithNewStateConnecting(child, priority, oldState) + default: + // New state is Shutdown, should never happen. Don't forward. + } +} + +// handlePriorityWithNewStateReady handles state Ready from a higher or equal +// priority. +// +// An update with state Ready: +// - If it's from higher priority: +// - Switch to this priority +// - Forward the update +// - If it's from priorityInUse: +// - Forward only +// +// Caller must make sure priorityInUse is not higher than priority. +// +// Caller must hold mu. +func (b *priorityBalancer) handlePriorityWithNewStateReady(child *childBalancer, priority int) { + // If one priority higher or equal to priorityInUse goes Ready, stop the + // init timer. If update is from higher than priorityInUse, priorityInUse + // will be closed, and the init timer will become useless. + b.stopPriorityInitTimer() + + // priorityInUse is lower than this priority, switch to this. + if b.priorityInUse > priority { + b.logger.Infof("Switching priority from %v to %v, because latter became Ready", b.priorityInUse, priority) + b.switchToChild(child, priority) + } + // Forward the update since it's READY. + b.cc.UpdateState(child.state) +} + +// handlePriorityWithNewStateTransientFailure handles state TransientFailure +// from a higher or equal priority. +// +// An update with state TransientFailure: +// - If it's from a higher priority: +// - Do not forward, and do nothing +// - If it's from priorityInUse: +// - If there's no lower: +// - Forward and do nothing else +// - If there's a lower priority: +// - Switch to the lower +// - Forward the lower child's state +// - Do NOT forward this update +// +// Caller must make sure priorityInUse is not higher than priority. +// +// Caller must hold mu. +func (b *priorityBalancer) handlePriorityWithNewStateTransientFailure(child *childBalancer, priority int) { + // priorityInUse is lower than this priority, do nothing. + if b.priorityInUse > priority { + return + } + // priorityInUse sends a failure. Stop its init timer. + b.stopPriorityInitTimer() + priorityNext := priority + 1 + if priorityNext >= len(b.priorities) { + // Forward this update. + b.cc.UpdateState(child.state) + return + } + b.logger.Infof("Switching priority from %v to %v, because former became TransientFailure", priority, priorityNext) + nameNext := b.priorities[priorityNext] + childNext := b.children[nameNext] + b.switchToChild(childNext, priorityNext) + b.cc.UpdateState(childNext.state) + childNext.sendUpdate() +} + +// handlePriorityWithNewStateConnecting handles state Connecting from a higher +// than or equal priority. +// +// An update with state Connecting: +// - If it's from a higher priority +// - Do nothing +// - If it's from priorityInUse, the behavior depends on previous state. +// +// When new state is Connecting, the behavior depends on previous state. If the +// previous state was Ready, this is a transition out from Ready to Connecting. +// Assuming there are multiple backends in the same priority, this mean we are +// in a bad situation and we should failover to the next priority (Side note: +// the current connectivity state aggregating algorithm (e.g. round-robin) is +// not handling this right, because if many backends all go from Ready to +// Connecting, the overall situation is more like TransientFailure, not +// Connecting). +// +// If the previous state was Idle, we don't do anything special with failure, +// and simply forward the update. The init timer should be in process, will +// handle failover if it timeouts. If the previous state was TransientFailure, +// we do not forward, because the lower priority is in use. +// +// Caller must make sure priorityInUse is not higher than priority. +// +// Caller must hold mu. +func (b *priorityBalancer) handlePriorityWithNewStateConnecting(child *childBalancer, priority int, oldState connectivity.State) { + // priorityInUse is lower than this priority, do nothing. + if b.priorityInUse > priority { + return + } + + switch oldState { + case connectivity.Ready: + // Handling transition from Ready to Connecting, is same as handling + // TransientFailure. There's no need to stop the init timer, because it + // should have been stopped when state turned Ready. + priorityNext := priority + 1 + if priorityNext >= len(b.priorities) { + // Forward this update. + b.cc.UpdateState(child.state) + return + } + b.logger.Infof("Switching priority from %v to %v, because former became TransientFailure", priority, priorityNext) + nameNext := b.priorities[priorityNext] + childNext := b.children[nameNext] + b.switchToChild(childNext, priorityNext) + b.cc.UpdateState(childNext.state) + childNext.sendUpdate() + case connectivity.Idle: + b.cc.UpdateState(child.state) + default: + // Old state is Connecting, TransientFailure or Shutdown. Don't forward. + } +} diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go new file mode 100644 index 0000000000..cb337c36e5 --- /dev/null +++ b/xds/balancer/priority/config.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "encoding/json" + "fmt" + + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/serviceconfig" +) + +// Child is a child of priority balancer. +type Child struct { + Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"` + IgnoreReresolutionRequests bool `json:"ignoreReresolutionRequests,omitempty"` +} + +// LBConfig represents priority balancer's config. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + // Children is a map from the child balancer names to their configs. Child + // names can be found in field Priorities. + Children map[string]*Child `json:"children,omitempty"` + // Priorities is a list of child balancer names. They are sorted from + // highest priority to low. The type/config for each child can be found in + // field Children, with the balancer name as the key. + Priorities []string `json:"priorities,omitempty"` +} + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + + prioritiesSet := make(map[string]bool) + for _, name := range cfg.Priorities { + if _, ok := cfg.Children[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Priorities field (%v) is not found in Children field (%+v)", name, cfg.Priorities, cfg.Children) + } + prioritiesSet[name] = true + } + for name := range cfg.Children { + if _, ok := prioritiesSet[name]; !ok { + return nil, fmt.Errorf("LB policy name %q found in Children field (%v) is not found in Priorities field (%+v)", name, cfg.Children, cfg.Priorities) + } + } + return &cfg, nil +} diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go new file mode 100644 index 0000000000..67045bf11e --- /dev/null +++ b/xds/balancer/priority/config_test.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "testing" + + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer/roundrobin" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "child not found", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "child not used", + js: `{ + "priorities": ["child-1", "child-2"], + "children": { + "child-1": {"config": [{"round_robin":{}}]}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + wantErr: true, + }, + { + name: "good", + js: `{ + "priorities": ["child-1", "child-2", "child-3"], + "children": { + "child-1": {"config": [{"round_robin":{}}], "ignoreReresolutionRequests": true}, + "child-2": {"config": [{"round_robin":{}}]}, + "child-3": {"config": [{"round_robin":{}}]} + } +} + `, + want: &LBConfig{ + Children: map[string]*Child{ + "child-1": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + IgnoreReresolutionRequests: true, + }, + "child-2": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + "child-3": { + Config: &internalserviceconfig.BalancerConfig{ + Name: roundrobin.Name, + }, + }, + }, + Priorities: []string{"child-1", "child-2", "child-3"}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got = %v, want %v, diff: %s", got, tt.want, diff) + } + }) + } +} diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go new file mode 100644 index 0000000000..9a9f477726 --- /dev/null +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "sync/atomic" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" +) + +type ignoreResolveNowBalancerBuilder struct { + balancer.Builder + ignoreResolveNow *uint32 +} + +// If `ignore` is true, all `ResolveNow()` from the balancer built from this +// builder will be ignored. +// +// `ignore` can be updated later by `updateIgnoreResolveNow`, and the update +// will be propagated to all the old and new balancers built with this. +func newIgnoreResolveNowBalancerBuilder(bb balancer.Builder, ignore bool) *ignoreResolveNowBalancerBuilder { + ret := &ignoreResolveNowBalancerBuilder{ + Builder: bb, + ignoreResolveNow: new(uint32), + } + ret.updateIgnoreResolveNow(ignore) + return ret +} + +func (irnbb *ignoreResolveNowBalancerBuilder) updateIgnoreResolveNow(b bool) { + if b { + atomic.StoreUint32(irnbb.ignoreResolveNow, 1) + return + } + atomic.StoreUint32(irnbb.ignoreResolveNow, 0) + +} + +func (irnbb *ignoreResolveNowBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + return irnbb.Builder.Build(&ignoreResolveNowClientConn{ + ClientConn: cc, + ignoreResolveNow: irnbb.ignoreResolveNow, + }, opts) +} + +type ignoreResolveNowClientConn struct { + balancer.ClientConn + ignoreResolveNow *uint32 +} + +func (i ignoreResolveNowClientConn) ResolveNow(o resolver.ResolveNowOptions) { + if atomic.LoadUint32(i.ignoreResolveNow) != 0 { + return + } + i.ClientConn.ResolveNow(o) +} diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go new file mode 100644 index 0000000000..53d2af57be --- /dev/null +++ b/xds/balancer/priority/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[priority-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *priorityBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/priority/utils.go b/xds/balancer/priority/utils.go new file mode 100644 index 0000000000..45fbe76443 --- /dev/null +++ b/xds/balancer/priority/utils.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +func equalStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go new file mode 100644 index 0000000000..c80a89b080 --- /dev/null +++ b/xds/balancer/priority/utils_test.go @@ -0,0 +1,62 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package priority + +import "testing" + +func TestCompareStringSlice(t *testing.T) { + tests := []struct { + name string + a []string + b []string + want bool + }{ + { + name: "equal", + a: []string{"a", "b"}, + b: []string{"a", "b"}, + want: true, + }, + { + name: "not equal", + a: []string{"a", "b"}, + b: []string{"a", "b", "c"}, + want: false, + }, + { + name: "both empty", + a: nil, + b: nil, + want: true, + }, + { + name: "one empty", + a: []string{"a", "b"}, + b: nil, + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := equalStringSlice(tt.a, tt.b); got != tt.want { + t.Errorf("equalStringSlice(%v, %v) = %v, want %v", tt.a, tt.b, got, tt.want) + } + }) + } +} diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go new file mode 100644 index 0000000000..5cb4aab3d9 --- /dev/null +++ b/xds/balancer/ringhash/config.go @@ -0,0 +1,56 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/serviceconfig" +) + +// LBConfig is the balancer config for ring_hash balancer. +type LBConfig struct { + serviceconfig.LoadBalancingConfig `json:"-"` + + MinRingSize uint64 `json:"minRingSize,omitempty"` + MaxRingSize uint64 `json:"maxRingSize,omitempty"` +} + +const ( + defaultMinSize = 1024 + defaultMaxSize = 8 * 1024 * 1024 // 8M +) + +func parseConfig(c json.RawMessage) (*LBConfig, error) { + var cfg LBConfig + if err := json.Unmarshal(c, &cfg); err != nil { + return nil, err + } + if cfg.MinRingSize == 0 { + cfg.MinRingSize = defaultMinSize + } + if cfg.MaxRingSize == 0 { + cfg.MaxRingSize = defaultMaxSize + } + if cfg.MinRingSize > cfg.MaxRingSize { + return nil, fmt.Errorf("min %v is greater than max %v", cfg.MinRingSize, cfg.MaxRingSize) + } + return &cfg, nil +} diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go new file mode 100644 index 0000000000..a2a966dc31 --- /dev/null +++ b/xds/balancer/ringhash/config_test.go @@ -0,0 +1,68 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseConfig(t *testing.T) { + tests := []struct { + name string + js string + want *LBConfig + wantErr bool + }{ + { + name: "OK", + js: `{"minRingSize": 1, "maxRingSize": 2}`, + want: &LBConfig{MinRingSize: 1, MaxRingSize: 2}, + }, + { + name: "OK with default min", + js: `{"maxRingSize": 2000}`, + want: &LBConfig{MinRingSize: defaultMinSize, MaxRingSize: 2000}, + }, + { + name: "OK with default max", + js: `{"minRingSize": 2000}`, + want: &LBConfig{MinRingSize: 2000, MaxRingSize: defaultMaxSize}, + }, + { + name: "min greater than max", + js: `{"minRingSize": 10, "maxRingSize": 2}`, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := parseConfig([]byte(tt.js)) + if (err != nil) != tt.wantErr { + t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr) + return + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Errorf("parseConfig() got unexpected output, diff (-got +want): %v", diff) + } + }) + } +} diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go new file mode 100644 index 0000000000..584f99fe8c --- /dev/null +++ b/xds/balancer/ringhash/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[ring-hash-lb %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *ringhashBalancer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go new file mode 100644 index 0000000000..5e0fab0365 --- /dev/null +++ b/xds/balancer/ringhash/picker.go @@ -0,0 +1,154 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" +) + +type picker struct { + ring *ring + logger *grpclog.PrefixLogger +} + +func newPicker(ring *ring, logger *grpclog.PrefixLogger) *picker { + return &picker{ring: ring, logger: logger} +} + +// handleRICSResult is the return type of handleRICS. It's needed to wrap the +// returned error from Pick() in a struct. With this, if the return values are +// `balancer.PickResult, error, bool`, linter complains because error is not the +// last return value. +type handleRICSResult struct { + pr balancer.PickResult + err error +} + +// handleRICS generates pick result if the entry is in Ready, Idle, Connecting +// or Shutdown. TransientFailure will be handled specifically after this +// function returns. +// +// The first return value indicates if the state is in Ready, Idle, Connecting +// or Shutdown. If it's true, the PickResult and error should be returned from +// Pick() as is. +func (p *picker) handleRICS(e *ringEntry) (handleRICSResult, bool) { + switch state := e.sc.effectiveState(); state { + case connectivity.Ready: + return handleRICSResult{pr: balancer.PickResult{SubConn: e.sc.sc}}, true + case connectivity.Idle: + // Trigger Connect() and queue the pick. + e.sc.queueConnect() + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + case connectivity.Connecting: + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + case connectivity.TransientFailure: + // Return ok==false, so TransientFailure will be handled afterwards. + return handleRICSResult{}, false + case connectivity.Shutdown: + // Shutdown can happen in a race where the old picker is called. A new + // picker should already be sent. + return handleRICSResult{err: balancer.ErrNoSubConnAvailable}, true + default: + // Should never reach this. All the connectivity states are already + // handled in the cases. + p.logger.Errorf("SubConn has undefined connectivity state: %v", state) + return handleRICSResult{err: status.Errorf(codes.Unavailable, "SubConn has undefined connectivity state: %v", state)}, true + } +} + +func (p *picker) Pick(info balancer.PickInfo) (balancer.PickResult, error) { + e := p.ring.pick(getRequestHash(info.Ctx)) + if hr, ok := p.handleRICS(e); ok { + return hr.pr, hr.err + } + // ok was false, the entry is in transient failure. + return p.handleTransientFailure(e) +} + +func (p *picker) handleTransientFailure(e *ringEntry) (balancer.PickResult, error) { + // Queue a connect on the first picked SubConn. + e.sc.queueConnect() + + // Find next entry in the ring, skipping duplicate SubConns. + e2 := nextSkippingDuplicates(p.ring, e) + if e2 == nil { + // There's no next entry available, fail the pick. + return balancer.PickResult{}, fmt.Errorf("the only SubConn is in Transient Failure") + } + + // For the second SubConn, also check Ready/Idle/Connecting as if it's the + // first entry. + if hr, ok := p.handleRICS(e2); ok { + return hr.pr, hr.err + } + + // The second SubConn is also in TransientFailure. Queue a connect on it. + e2.sc.queueConnect() + + // If it gets here, this is after the second SubConn, and the second SubConn + // was in TransientFailure. + // + // Loop over all other SubConns: + // - If all SubConns so far are all TransientFailure, trigger Connect() on + // the TransientFailure SubConns, and keep going. + // - If there's one SubConn that's not in TransientFailure, keep checking + // the remaining SubConns (in case there's a Ready, which will be returned), + // but don't not trigger Connect() on the other SubConns. + var firstNonFailedFound bool + for ee := nextSkippingDuplicates(p.ring, e2); ee != e; ee = nextSkippingDuplicates(p.ring, ee) { + scState := ee.sc.effectiveState() + if scState == connectivity.Ready { + return balancer.PickResult{SubConn: ee.sc.sc}, nil + } + if firstNonFailedFound { + continue + } + if scState == connectivity.TransientFailure { + // This will queue a connect. + ee.sc.queueConnect() + continue + } + // This is a SubConn in a non-failure state. We continue to check the + // other SubConns, but remember that there was a non-failed SubConn + // seen. After this, Pick() will never trigger any SubConn to Connect(). + firstNonFailedFound = true + if scState == connectivity.Idle { + // This is the first non-failed SubConn, and it is in a real Idle + // state. Trigger it to Connect(). + ee.sc.queueConnect() + } + } + return balancer.PickResult{}, fmt.Errorf("no connection is Ready") +} + +func nextSkippingDuplicates(ring *ring, entry *ringEntry) *ringEntry { + for next := ring.next(entry); next != entry; next = ring.next(next) { + if next.sc != entry.sc { + return next + } + } + // There's no qualifying next entry. + return nil +} diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go new file mode 100644 index 0000000000..68e844cfb4 --- /dev/null +++ b/xds/balancer/ringhash/ring.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + "math" + "sort" + "strconv" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" +) + +type ring struct { + items []*ringEntry +} + +type subConnWithWeight struct { + sc *subConn + weight float64 +} + +type ringEntry struct { + idx int + hash uint64 + sc *subConn +} + +// newRing creates a ring from the subConns. The ring size is limited by the +// passed in max/min. +// +// ring entries will be created for each subConn, and subConn with high weight +// (specified by the address) may have multiple entries. +// +// For example, for subConns with weights {a:3, b:3, c:4}, a generated ring of +// size 10 could be: +// - {idx:0 hash:3689675255460411075 b} +// - {idx:1 hash:4262906501694543955 c} +// - {idx:2 hash:5712155492001633497 c} +// - {idx:3 hash:8050519350657643659 b} +// - {idx:4 hash:8723022065838381142 b} +// - {idx:5 hash:11532782514799973195 a} +// - {idx:6 hash:13157034721563383607 c} +// - {idx:7 hash:14468677667651225770 c} +// - {idx:8 hash:17336016884672388720 a} +// - {idx:9 hash:18151002094784932496 a} +// +// To pick from a ring, a binary search will be done for the given target hash, +// and first item with hash >= given hash will be returned. +func newRing(subConns map[resolver.Address]*subConn, minRingSize, maxRingSize uint64) (*ring, error) { + // https://github.com/envoyproxy/envoy/blob/765c970f06a4c962961a0e03a467e165b276d50f/source/common/upstream/ring_hash_lb.cc#L114 + normalizedWeights, minWeight, err := normalizeWeights(subConns) + if err != nil { + return nil, err + } + // Normalized weights for {3,3,4} is {0.3,0.3,0.4}. + + // Scale up the size of the ring such that the least-weighted host gets a + // whole number of hashes on the ring. + // + // Note that size is limited by the input max/min. + scale := math.Min(math.Ceil(minWeight*float64(minRingSize))/minWeight, float64(maxRingSize)) + ringSize := math.Ceil(scale) + items := make([]*ringEntry, 0, int(ringSize)) + + // For each entry, scale*weight nodes are generated in the ring. + // + // Not all of these are whole numbers. E.g. for weights {a:3,b:3,c:4}, if + // ring size is 7, scale is 6.66. The numbers of nodes will be + // {a,a,b,b,c,c,c}. + // + // A hash is generated for each item, and later the results will be sorted + // based on the hash. + var ( + idx int + targetIdx float64 + ) + for _, scw := range normalizedWeights { + targetIdx += scale * scw.weight + for float64(idx) < targetIdx { + h := xxhash.Sum64String(scw.sc.addr + strconv.Itoa(len(items))) + items = append(items, &ringEntry{idx: idx, hash: h, sc: scw.sc}) + idx++ + } + } + + // Sort items based on hash, to prepare for binary search. + sort.Slice(items, func(i, j int) bool { return items[i].hash < items[j].hash }) + for i, ii := range items { + ii.idx = i + } + return &ring{items: items}, nil +} + +// normalizeWeights divides all the weights by the sum, so that the total weight +// is 1. +func normalizeWeights(subConns map[resolver.Address]*subConn) (_ []subConnWithWeight, min float64, _ error) { + if len(subConns) == 0 { + return nil, 0, fmt.Errorf("number of subconns is 0") + } + var weightSum uint32 + for a := range subConns { + // The address weight was moved from attributes to the Metadata field. + // This is necessary (all the attributes need to be stripped) for the + // balancer to detect identical {address+weight} combination. + weightSum += a.Metadata.(uint32) + } + if weightSum == 0 { + return nil, 0, fmt.Errorf("total weight of all subconns is 0") + } + weightSumF := float64(weightSum) + ret := make([]subConnWithWeight, 0, len(subConns)) + min = math.MaxFloat64 + for a, sc := range subConns { + nw := float64(a.Metadata.(uint32)) / weightSumF + ret = append(ret, subConnWithWeight{sc: sc, weight: nw}) + if nw < min { + min = nw + } + } + // Sort the addresses to return consistent results. + // + // Note: this might not be necessary, but this makes sure the ring is + // consistent as long as the addresses are the same, for example, in cases + // where an address is added and then removed, the RPCs will still pick the + // same old SubConn. + sort.Slice(ret, func(i, j int) bool { return ret[i].sc.addr < ret[j].sc.addr }) + return ret, min, nil +} + +// pick does a binary search. It returns the item with smallest index i that +// r.items[i].hash >= h. +func (r *ring) pick(h uint64) *ringEntry { + i := sort.Search(len(r.items), func(i int) bool { return r.items[i].hash >= h }) + if i == len(r.items) { + // If not found, and h is greater than the largest hash, return the + // first item. + i = 0 + } + return r.items[i] +} + +// next returns the next entry. +func (r *ring) next(e *ringEntry) *ringEntry { + return r.items[(e.idx+1)%len(r.items)] +} diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go new file mode 100644 index 0000000000..2d664e05bb --- /dev/null +++ b/xds/balancer/ringhash/ring_test.go @@ -0,0 +1,113 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import ( + "fmt" + "math" + "testing" + + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" +) + +func testAddr(addr string, weight uint32) resolver.Address { + return resolver.Address{Addr: addr, Metadata: weight} +} + +func TestRingNew(t *testing.T) { + testAddrs := []resolver.Address{ + testAddr("a", 3), + testAddr("b", 3), + testAddr("c", 4), + } + var totalWeight float64 = 10 + testSubConnMap := map[resolver.Address]*subConn{ + testAddr("a", 3): {addr: "a"}, + testAddr("b", 3): {addr: "b"}, + testAddr("c", 4): {addr: "c"}, + } + for _, min := range []uint64{3, 4, 6, 8} { + for _, max := range []uint64{20, 8} { + t.Run(fmt.Sprintf("size-min-%v-max-%v", min, max), func(t *testing.T) { + r, _ := newRing(testSubConnMap, min, max) + totalCount := len(r.items) + if totalCount < int(min) || totalCount > int(max) { + t.Fatalf("unexpect size %v, want min %v, max %v", totalCount, min, max) + } + for _, a := range testAddrs { + var count int + for _, ii := range r.items { + if ii.sc.addr == a.Addr { + count++ + } + } + got := float64(count) / float64(totalCount) + want := float64(a.Metadata.(uint32)) / totalWeight + if !equalApproximately(got, want) { + t.Fatalf("unexpected item weight in ring: %v != %v", got, want) + } + } + }) + } + } +} + +func equalApproximately(x, y float64) bool { + delta := math.Abs(x - y) + mean := math.Abs(x+y) / 2.0 + return delta/mean < 0.25 +} + +func TestRingPick(t *testing.T) { + r, _ := newRing(map[resolver.Address]*subConn{ + {Addr: "a", Metadata: uint32(3)}: {addr: "a"}, + {Addr: "b", Metadata: uint32(3)}: {addr: "b"}, + {Addr: "c", Metadata: uint32(4)}: {addr: "c"}, + }, 10, 20) + for _, h := range []uint64{xxhash.Sum64String("1"), xxhash.Sum64String("2"), xxhash.Sum64String("3"), xxhash.Sum64String("4")} { + t.Run(fmt.Sprintf("picking-hash-%v", h), func(t *testing.T) { + e := r.pick(h) + var low uint64 + if e.idx > 0 { + low = r.items[e.idx-1].hash + } + high := e.hash + // h should be in [low, high). + if h < low || h >= high { + t.Fatalf("unexpected item picked, hash: %v, low: %v, high: %v", h, low, high) + } + }) + } +} + +func TestRingNext(t *testing.T) { + r, _ := newRing(map[resolver.Address]*subConn{ + {Addr: "a", Metadata: uint32(3)}: {addr: "a"}, + {Addr: "b", Metadata: uint32(3)}: {addr: "b"}, + {Addr: "c", Metadata: uint32(4)}: {addr: "c"}, + }, 10, 20) + + for _, e := range r.items { + ne := r.next(e) + if ne.idx != (e.idx+1)%len(r.items) { + t.Fatalf("next(%+v) returned unexpected %+v", e, ne) + } + } +} diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go new file mode 100644 index 0000000000..947a20ae20 --- /dev/null +++ b/xds/balancer/ringhash/ringhash.go @@ -0,0 +1,434 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package ringhash implements the ringhash balancer. +package ringhash + +import ( + "encoding/json" + "errors" + "fmt" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +// Name is the name of the ring_hash balancer. +const Name = "ring_hash_experimental" + +func init() { + balancer.Register(bb{}) +} + +type bb struct{} + +func (bb) Build(cc balancer.ClientConn, bOpts balancer.BuildOptions) balancer.Balancer { + b := &ringhashBalancer{ + cc: cc, + subConns: make(map[resolver.Address]*subConn), + scStates: make(map[balancer.SubConn]*subConn), + csEvltr: &connectivityStateEvaluator{}, + } + b.logger = prefixLogger(b) + b.logger.Infof("Created") + return b +} + +func (bb) Name() string { + return Name +} + +func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, error) { + return parseConfig(c) +} + +type subConn struct { + addr string + sc balancer.SubConn + + mu sync.RWMutex + // This is the actual state of this SubConn (as updated by the ClientConn). + // The effective state can be different, see comment of attemptedToConnect. + state connectivity.State + // failing is whether this SubConn is in a failing state. A subConn is + // considered to be in a failing state if it was previously in + // TransientFailure. + // + // This affects the effective connectivity state of this SubConn, e.g. + // - if the actual state is Idle or Connecting, but this SubConn is failing, + // the effective state is TransientFailure. + // + // This is used in pick(). E.g. if a subConn is Idle, but has failing as + // true, pick() will + // - consider this SubConn as TransientFailure, and check the state of the + // next SubConn. + // - trigger Connect() (note that normally a SubConn in real + // TransientFailure cannot Connect()) + // + // A subConn starts in non-failing (failing is false). A transition to + // TransientFailure sets failing to true (and it stays true). A transition + // to Ready sets failing to false. + failing bool + // connectQueued is true if a Connect() was queued for this SubConn while + // it's not in Idle (most likely was in TransientFailure). A Connect() will + // be triggered on this SubConn when it turns Idle. + // + // When connectivity state is updated to Idle for this SubConn, if + // connectQueued is true, Connect() will be called on the SubConn. + connectQueued bool +} + +// setState updates the state of this SubConn. +// +// It also handles the queued Connect(). If the new state is Idle, and a +// Connect() was queued, this SubConn will be triggered to Connect(). +func (sc *subConn) setState(s connectivity.State) { + sc.mu.Lock() + defer sc.mu.Unlock() + switch s { + case connectivity.Idle: + // Trigger Connect() if new state is Idle, and there is a queued connect. + if sc.connectQueued { + sc.connectQueued = false + sc.sc.Connect() + } + case connectivity.Connecting: + // Clear connectQueued if the SubConn isn't failing. This state + // transition is unlikely to happen, but handle this just in case. + sc.connectQueued = false + case connectivity.Ready: + // Clear connectQueued if the SubConn isn't failing. This state + // transition is unlikely to happen, but handle this just in case. + sc.connectQueued = false + // Set to a non-failing state. + sc.failing = false + case connectivity.TransientFailure: + // Set to a failing state. + sc.failing = true + } + sc.state = s +} + +// effectiveState returns the effective state of this SubConn. It can be +// different from the actual state, e.g. Idle while the subConn is failing is +// considered TransientFailure. Read comment of field failing for other cases. +func (sc *subConn) effectiveState() connectivity.State { + sc.mu.RLock() + defer sc.mu.RUnlock() + if sc.failing && (sc.state == connectivity.Idle || sc.state == connectivity.Connecting) { + return connectivity.TransientFailure + } + return sc.state +} + +// queueConnect sets a boolean so that when the SubConn state changes to Idle, +// it's Connect() will be triggered. If the SubConn state is already Idle, it +// will just call Connect(). +func (sc *subConn) queueConnect() { + sc.mu.Lock() + defer sc.mu.Unlock() + if sc.state == connectivity.Idle { + sc.sc.Connect() + return + } + // Queue this connect, and when this SubConn switches back to Idle (happens + // after backoff in TransientFailure), it will Connect(). + sc.connectQueued = true +} + +type ringhashBalancer struct { + cc balancer.ClientConn + logger *grpclog.PrefixLogger + + config *LBConfig + + subConns map[resolver.Address]*subConn // `attributes` is stripped from the keys of this map (the addresses) + scStates map[balancer.SubConn]*subConn + + // ring is always in sync with subConns. When subConns change, a new ring is + // generated. Note that address weights updates (they are keys in the + // subConns map) also regenerates the ring. + ring *ring + picker balancer.Picker + csEvltr *connectivityStateEvaluator + state connectivity.State + + resolverErr error // the last error reported by the resolver; cleared on successful resolution + connErr error // the last connection error; cleared upon leaving TransientFailure +} + +// updateAddresses creates new SubConns and removes SubConns, based on the +// address update. +// +// The return value is whether the new address list is different from the +// previous. True if +// - an address was added +// - an address was removed +// - an address's weight was updated +// +// Note that this function doesn't trigger SubConn connecting, so all the new +// SubConn states are Idle. +func (b *ringhashBalancer) updateAddresses(addrs []resolver.Address) bool { + var addrsUpdated bool + // addrsSet is the set converted from addrs, it's used for quick lookup of + // an address. + // + // Addresses in this map all have attributes stripped, but metadata set to + // the weight. So that weight change can be detected. + // + // TODO: this won't be necessary if there are ways to compare address + // attributes. + addrsSet := make(map[resolver.Address]struct{}) + for _, a := range addrs { + aNoAttrs := a + // Strip attributes but set Metadata to the weight. + aNoAttrs.Attributes = nil + w := weightedroundrobin.GetAddrInfo(a).Weight + if w == 0 { + // If weight is not set, use 1. + w = 1 + } + aNoAttrs.Metadata = w + addrsSet[aNoAttrs] = struct{}{} + if scInfo, ok := b.subConns[aNoAttrs]; !ok { + // When creating SubConn, the original address with attributes is + // passed through. So that connection configurations in attributes + // (like creds) will be used. + sc, err := b.cc.NewSubConn([]resolver.Address{a}, balancer.NewSubConnOptions{HealthCheckEnabled: true}) + if err != nil { + logger.Warningf("base.baseBalancer: failed to create new SubConn: %v", err) + continue + } + scs := &subConn{addr: a.Addr, sc: sc} + scs.setState(connectivity.Idle) + b.state = b.csEvltr.recordTransition(connectivity.Shutdown, connectivity.Idle) + b.subConns[aNoAttrs] = scs + b.scStates[sc] = scs + addrsUpdated = true + } else { + // Always update the subconn's address in case the attributes + // changed. The SubConn does a reflect.DeepEqual of the new and old + // addresses. So this is a noop if the current address is the same + // as the old one (including attributes). + b.subConns[aNoAttrs] = scInfo + b.cc.UpdateAddresses(scInfo.sc, []resolver.Address{a}) + } + } + for a, scInfo := range b.subConns { + // a was removed by resolver. + if _, ok := addrsSet[a]; !ok { + b.cc.RemoveSubConn(scInfo.sc) + delete(b.subConns, a) + addrsUpdated = true + // Keep the state of this sc in b.scStates until sc's state becomes Shutdown. + // The entry will be deleted in UpdateSubConnState. + } + } + return addrsUpdated +} + +func (b *ringhashBalancer) UpdateClientConnState(s balancer.ClientConnState) error { + b.logger.Infof("Received update from resolver, balancer config: %+v", pretty.ToJSON(s.BalancerConfig)) + if b.config == nil { + newConfig, ok := s.BalancerConfig.(*LBConfig) + if !ok { + return fmt.Errorf("unexpected balancer config with type: %T", s.BalancerConfig) + } + b.config = newConfig + } + + // Successful resolution; clear resolver error and ensure we return nil. + b.resolverErr = nil + if b.updateAddresses(s.ResolverState.Addresses) { + // If addresses were updated, no matter whether it resulted in SubConn + // creation/deletion, or just weight update, we will need to regenerate + // the ring. + var err error + b.ring, err = newRing(b.subConns, b.config.MinRingSize, b.config.MaxRingSize) + if err != nil { + panic(err) + } + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + } + + // If resolver state contains no addresses, return an error so ClientConn + // will trigger re-resolve. Also records this as an resolver error, so when + // the overall state turns transient failure, the error message will have + // the zero address information. + if len(s.ResolverState.Addresses) == 0 { + b.ResolverError(errors.New("produced zero addresses")) + return balancer.ErrBadResolverState + } + return nil +} + +func (b *ringhashBalancer) ResolverError(err error) { + b.resolverErr = err + if len(b.subConns) == 0 { + b.state = connectivity.TransientFailure + } + + if b.state != connectivity.TransientFailure { + // The picker will not change since the balancer does not currently + // report an error. + return + } + b.regeneratePicker() + b.cc.UpdateState(balancer.State{ + ConnectivityState: b.state, + Picker: b.picker, + }) +} + +// UpdateSubConnState updates the per-SubConn state stored in the ring, and also +// the aggregated state. +// +// It triggers an update to cc when: +// - the new state is TransientFailure, to update the error message +// - it's possible that this is a noop, but sending an extra update is easier +// than comparing errors +// - the aggregated state is changed +// - the same picker will be sent again, but this update may trigger a re-pick +// for some RPCs. +func (b *ringhashBalancer) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + s := state.ConnectivityState + b.logger.Infof("handle SubConn state change: %p, %v", sc, s) + scs, ok := b.scStates[sc] + if !ok { + b.logger.Infof("got state changes for an unknown SubConn: %p, %v", sc, s) + return + } + oldSCState := scs.effectiveState() + scs.setState(s) + newSCState := scs.effectiveState() + + var sendUpdate bool + oldBalancerState := b.state + b.state = b.csEvltr.recordTransition(oldSCState, newSCState) + if oldBalancerState != b.state { + sendUpdate = true + } + + switch s { + case connectivity.Idle: + // When the overall state is TransientFailure, this will never get picks + // if there's a lower priority. Need to keep the SubConns connecting so + // there's a chance it will recover. + if b.state == connectivity.TransientFailure { + scs.queueConnect() + } + // No need to send an update. No queued RPC can be unblocked. If the + // overall state changed because of this, sendUpdate is already true. + case connectivity.Connecting: + // No need to send an update. No queued RPC can be unblocked. If the + // overall state changed because of this, sendUpdate is already true. + case connectivity.Ready: + // Resend the picker, there's no need to regenerate the picker because + // the ring didn't change. + sendUpdate = true + case connectivity.TransientFailure: + // Save error to be reported via picker. + b.connErr = state.ConnectionError + // Regenerate picker to update error message. + b.regeneratePicker() + sendUpdate = true + case connectivity.Shutdown: + // When an address was removed by resolver, b called RemoveSubConn but + // kept the sc's state in scStates. Remove state for this sc here. + delete(b.scStates, sc) + } + + if sendUpdate { + b.cc.UpdateState(balancer.State{ConnectivityState: b.state, Picker: b.picker}) + } +} + +// mergeErrors builds an error from the last connection error and the last +// resolver error. Must only be called if b.state is TransientFailure. +func (b *ringhashBalancer) mergeErrors() error { + // connErr must always be non-nil unless there are no SubConns, in which + // case resolverErr must be non-nil. + if b.connErr == nil { + return fmt.Errorf("last resolver error: %v", b.resolverErr) + } + if b.resolverErr == nil { + return fmt.Errorf("last connection error: %v", b.connErr) + } + return fmt.Errorf("last connection error: %v; last resolver error: %v", b.connErr, b.resolverErr) +} + +func (b *ringhashBalancer) regeneratePicker() { + if b.state == connectivity.TransientFailure { + b.picker = base.NewErrPicker(b.mergeErrors()) + return + } + b.picker = newPicker(b.ring, b.logger) +} + +func (b *ringhashBalancer) Close() {} + +// connectivityStateEvaluator takes the connectivity states of multiple SubConns +// and returns one aggregated connectivity state. +// +// It's not thread safe. +type connectivityStateEvaluator struct { + nums [5]uint64 +} + +// recordTransition records state change happening in subConn and based on that +// it evaluates what aggregated state should be. +// +// - If there is at least one subchannel in READY state, report READY. +// - If there are 2 or more subchannels in TRANSIENT_FAILURE state, report TRANSIENT_FAILURE. +// - If there is at least one subchannel in CONNECTING state, report CONNECTING. +// - If there is at least one subchannel in Idle state, report Idle. +// - Otherwise, report TRANSIENT_FAILURE. +// +// Note that if there are 1 connecting, 2 transient failure, the overall state +// is transient failure. This is because the second transient failure is a +// fallback of the first failing SubConn, and we want to report transient +// failure to failover to the lower priority. +func (cse *connectivityStateEvaluator) recordTransition(oldState, newState connectivity.State) connectivity.State { + // Update counters. + for idx, state := range []connectivity.State{oldState, newState} { + updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new. + cse.nums[state] += updateVal + } + + if cse.nums[connectivity.Ready] > 0 { + return connectivity.Ready + } + if cse.nums[connectivity.TransientFailure] > 1 { + return connectivity.TransientFailure + } + if cse.nums[connectivity.Connecting] > 0 { + return connectivity.Connecting + } + if cse.nums[connectivity.Idle] > 0 { + return connectivity.Idle + } + return connectivity.TransientFailure +} diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go new file mode 100644 index 0000000000..92bb3ae5b7 --- /dev/null +++ b/xds/balancer/ringhash/util.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package ringhash + +import "context" + +type clusterKey struct{} + +func getRequestHash(ctx context.Context) uint64 { + requestHash, _ := ctx.Value(clusterKey{}).(uint64) + return requestHash +} + +// GetRequestHashForTesting returns the request hash in the context; to be used +// for testing only. +func GetRequestHashForTesting(ctx context.Context) uint64 { + return getRequestHash(ctx) +} + +// SetRequestHash adds the request hash to the context for use in Ring Hash Load +// Balancing. +func SetRequestHash(ctx context.Context, requestHash uint64) context.Context { + return context.WithValue(ctx, clusterKey{}, requestHash) +} diff --git a/xds/client/attributes.go b/xds/client/attributes.go new file mode 100644 index 0000000000..ecd3a13dd2 --- /dev/null +++ b/xds/client/attributes.go @@ -0,0 +1,60 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package client + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "google.golang.org/grpc/resolver" +) + +type clientKeyType string + +const clientKey = clientKeyType("grpc.xds.internal.client.Client") + +// XDSClient is a full fledged gRPC client which queries a set of discovery APIs +// (collectively termed as xDS) on a remote management server, to discover +// various dynamic resources. +type XDSClient interface { + WatchListener(string, func(resource.ListenerUpdate, error)) func() + WatchRouteConfig(string, func(resource.RouteConfigUpdate, error)) func() + WatchCluster(string, func(resource.ClusterUpdate, error)) func() + WatchEndpoints(clusterName string, edsCb func(resource.EndpointsUpdate, error)) (cancel func()) + ReportLoad(server string) (*load.Store, func()) + + DumpLDS() map[string]resource.UpdateWithMD + DumpRDS() map[string]resource.UpdateWithMD + DumpCDS() map[string]resource.UpdateWithMD + DumpEDS() map[string]resource.UpdateWithMD + + BootstrapConfig() *bootstrap.Config + Close() +} + +// FromResolverState returns the Client from state, or nil if not present. +func FromResolverState(state resolver.State) XDSClient { + cs, _ := state.Attributes.Value(clientKey).(XDSClient) + return cs +} + +// SetClient sets c in state and returns the new state. +func SetClient(state resolver.State, c XDSClient) resolver.State { + state.Attributes = state.Attributes.WithValue(clientKey, c) + return state +} diff --git a/xds/client/authority.go b/xds/client/authority.go new file mode 100644 index 0000000000..4cb486eb06 --- /dev/null +++ b/xds/client/authority.go @@ -0,0 +1,228 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "errors" + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +const federationScheme = "xdstp" + +// findAuthority returns the authority for this name. If it doesn't already +// exist, one will be created. +// +// Note that this doesn't always create new authority. authorities with the same +// config but different names are shared. +// +// The returned unref function must be called when the caller is done using this +// authority, without holding c.authorityMu. +// +// Caller must not hold c.authorityMu. +func (c *clientImpl) findAuthority(n *resource.Name) (_ *authority, unref func(), _ error) { + scheme, authority := n.Scheme, n.Authority + + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + if c.done.HasFired() { + return nil, nil, errors.New("the xds-client is closed") + } + + config := c.config.XDSServer + if scheme == federationScheme { + cfg, ok := c.config.Authorities[authority] + if !ok { + return nil, nil, fmt.Errorf("xds: failed to find authority %q", authority) + } + config = cfg.XDSServer + } + + a, err := c.newAuthority(config) + if err != nil { + return nil, nil, fmt.Errorf("xds: failed to connect to the control plane for authority %q: %v", authority, err) + } + // All returned authority from this function will be used by a watch, + // holding the ref here. + // + // Note that this must be done while c.authorityMu is held, to avoid the + // race that an authority is returned, but before the watch starts, the + // old last watch is canceled (in another goroutine), causing this + // authority to be removed, and then a watch will start on a removed + // authority. + // + // unref() will be done when the watch is canceled. + a.ref() + return a, func() { c.unrefAuthority(a) }, nil +} + +// newAuthority creates a new authority for the config. But before that, it +// checks the cache to see if an authority for this config already exists. +// +// caller must hold c.authorityMu +func (c *clientImpl) newAuthority(config *bootstrap.ServerConfig) (_ *authority, retErr error) { + // First check if there's already an authority for this config. If found, it + // means this authority is used by other watches (could be the same + // authority name, or a different authority name but the same server + // config). Return it. + configStr := config.String() + if a, ok := c.authorities[configStr]; ok { + return a, nil + } + // Second check if there's an authority in the idle cache. If found, it + // means this authority was created, but moved to the idle cache because the + // watch was canceled. Move it from idle cache to the authority cache, and + // return. + if old, ok := c.idleAuthorities.Remove(configStr); ok { + oldA, _ := old.(*authority) + if oldA != nil { + c.authorities[configStr] = oldA + return oldA, nil + } + } + + // Make a new authority since there's no existing authority for this config. + ret := &authority{config: config, pubsub: pubsub.New(c.watchExpiryTimeout, c.logger)} + defer func() { + if retErr != nil { + ret.close() + } + }() + ctr, err := newController(config, ret.pubsub, c.updateValidator, c.logger) + if err != nil { + return nil, err + } + ret.controller = ctr + // Add it to the cache, so it will be reused. + c.authorities[configStr] = ret + return ret, nil +} + +// unrefAuthority unrefs the authority. It also moves the authority to idle +// cache if it's ref count is 0. +// +// This function doesn't need to called explicitly. It's called by the returned +// unref from findAuthority(). +// +// Caller must not hold c.authorityMu. +func (c *clientImpl) unrefAuthority(a *authority) { + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + if a.unref() > 0 { + return + } + configStr := a.config.String() + delete(c.authorities, configStr) + c.idleAuthorities.Add(configStr, a, func() { + a.close() + }) +} + +// authority is a combination of pubsub and the controller for this authority. +// +// Note that it might make sense to use one pubsub for all the resources (for +// all the controllers). One downside is the handling of StoW APIs (LDS/CDS). +// These responses contain all the resources from that control plane, so pubsub +// will need to keep lists of resources from each control plane, to know what +// are removed. +type authority struct { + config *bootstrap.ServerConfig + pubsub *pubsub.Pubsub + controller controllerInterface + refCount int +} + +// caller must hold parent's authorityMu. +func (a *authority) ref() { + a.refCount++ +} + +// caller must hold parent's authorityMu. +func (a *authority) unref() int { + a.refCount-- + return a.refCount +} + +func (a *authority) close() { + if a.pubsub != nil { + a.pubsub.Close() + } + if a.controller != nil { + a.controller.Close() + } +} + +func (a *authority) watchListener(serviceName string, cb func(resource.ListenerUpdate, error)) (cancel func()) { + first, cancelF := a.pubsub.WatchListener(serviceName, cb) + if first { + a.controller.AddWatch(resource.ListenerResource, serviceName) + } + return func() { + if cancelF() { + a.controller.RemoveWatch(resource.ListenerResource, serviceName) + } + } +} + +func (a *authority) watchRouteConfig(routeName string, cb func(resource.RouteConfigUpdate, error)) (cancel func()) { + first, cancelF := a.pubsub.WatchRouteConfig(routeName, cb) + if first { + a.controller.AddWatch(resource.RouteConfigResource, routeName) + } + return func() { + if cancelF() { + a.controller.RemoveWatch(resource.RouteConfigResource, routeName) + } + } +} + +func (a *authority) watchCluster(clusterName string, cb func(resource.ClusterUpdate, error)) (cancel func()) { + first, cancelF := a.pubsub.WatchCluster(clusterName, cb) + if first { + a.controller.AddWatch(resource.ClusterResource, clusterName) + } + return func() { + if cancelF() { + a.controller.RemoveWatch(resource.ClusterResource, clusterName) + } + } +} + +func (a *authority) watchEndpoints(clusterName string, cb func(resource.EndpointsUpdate, error)) (cancel func()) { + first, cancelF := a.pubsub.WatchEndpoints(clusterName, cb) + if first { + a.controller.AddWatch(resource.EndpointsResource, clusterName) + } + return func() { + if cancelF() { + a.controller.RemoveWatch(resource.EndpointsResource, clusterName) + } + } +} + +func (a *authority) reportLoad(server string) (*load.Store, func()) { + return a.controller.ReportLoad(server) +} + +func (a *authority) dump(t resource.ResourceType) map[string]resource.UpdateWithMD { + return a.pubsub.Dump(t) +} diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go new file mode 100644 index 0000000000..646a858e37 --- /dev/null +++ b/xds/client/bootstrap/bootstrap.go @@ -0,0 +1,476 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package bootstrap provides the functionality to initialize certain aspects +// of an xDS client by reading a bootstrap file. +package bootstrap + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +) + +const ( + // The "server_features" field in the bootstrap file contains a list of + // features supported by the server. A value of "xds_v3" indicates that the + // server supports the v3 version of the xDS transport protocol. + serverFeaturesV3 = "xds_v3" + + // Type name for Google default credentials. + credsGoogleDefault = "google_default" + credsInsecure = "insecure" + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" +) + +var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version) + +// For overriding in unit tests. +var bootstrapFileReadFunc = ioutil.ReadFile + +// ServerConfig contains the configuration to connect to a server, including +// URI, creds, and transport API version (e.g. v2 or v3). +type ServerConfig struct { + // ServerURI is the management server to connect to. + // + // The bootstrap file contains an ordered list of xDS servers to contact for + // this authority. The first one is picked. + ServerURI string + // Creds contains the credentials to be used while talking to the xDS + // server, as a grpc.DialOption. + Creds grpc.DialOption + // CredsType is the type of the creds. It will be used to dedup servers. + CredsType string + // TransportAPI indicates the API version of xDS transport protocol to use. + // This describes the xDS gRPC endpoint and version of + // DiscoveryRequest/Response used on the wire. + TransportAPI version.TransportAPI + // NodeProto contains the Node proto to be used in xDS requests. The actual + // type depends on the transport protocol version used. + // + // Note that it's specified in the bootstrap globally for all the servers, + // but we keep it in each server config so that its type (e.g. *v2pb.Node or + // *v3pb.Node) is consistent with the transport API version. + NodeProto proto.Message +} + +// String returns the string representation of the ServerConfig. +// +// This string representation will be used as map keys in federation +// (`map[ServerConfig]authority`), so that the xDS ClientConn and stream will be +// shared by authorities with different names but the same server config. +// +// It covers (almost) all the fields so the string can represent the config +// content. It doesn't cover NodeProto because NodeProto isn't used by +// federation. +func (sc *ServerConfig) String() string { + var ver string + switch sc.TransportAPI { + case version.TransportV3: + ver = "xDSv3" + case version.TransportV2: + ver = "xDSv2" + } + return strings.Join([]string{sc.ServerURI, sc.CredsType, ver}, "-") +} + +// UnmarshalJSON takes the json data (a list of servers) and unmarshals the +// first one in the list. +func (sc *ServerConfig) UnmarshalJSON(data []byte) error { + var servers []*xdsServer + if err := json.Unmarshal(data, &servers); err != nil { + return fmt.Errorf("xds: json.Unmarshal(data) for field xds_servers failed during bootstrap: %v", err) + } + if len(servers) < 1 { + return fmt.Errorf("xds: bootstrap file parsing failed during bootstrap: file doesn't contain any management server to connect to") + } + xs := servers[0] + sc.ServerURI = xs.ServerURI + for _, cc := range xs.ChannelCreds { + // We stop at the first credential type that we support. + sc.CredsType = cc.Type + if cc.Type == credsGoogleDefault { + sc.Creds = grpc.WithCredentialsBundle(google.NewDefaultCredentials()) + break + } else if cc.Type == credsInsecure { + sc.Creds = grpc.WithTransportCredentials(insecure.NewCredentials()) + break + } + } + for _, f := range xs.ServerFeatures { + if f == serverFeaturesV3 { + sc.TransportAPI = version.TransportV3 + } + } + return nil +} + +// Authority contains configuration for an Authority for an xDS control plane +// server. See the Authorities field in the Config struct for how it's used. +type Authority struct { + // ClientListenerResourceNameTemplate is template for the name of the + // Listener resource to subscribe to for a gRPC client channel. Used only + // when the channel is created using an "xds:" URI with this authority name. + // + // The token "%s", if present in this string, will be replaced + // with %-encoded service authority (i.e., the path part of the target + // URI used to create the gRPC channel). + // + // Must start with "xdstp:///". If it does not, + // that is considered a bootstrap file parsing error. + // + // If not present in the bootstrap file, defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + ClientListenerResourceNameTemplate string + // XDSServer contains the management server and config to connect to for + // this authority. + XDSServer *ServerConfig +} + +// UnmarshalJSON implement json unmarshaller. +func (a *Authority) UnmarshalJSON(data []byte) error { + var jsonData map[string]json.RawMessage + if err := json.Unmarshal(data, &jsonData); err != nil { + return fmt.Errorf("xds: failed to parse authority: %v", err) + } + + for k, v := range jsonData { + switch k { + case "xds_servers": + if err := json.Unmarshal(v, &a.XDSServer); err != nil { + return fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "client_listener_resource_name_template": + if err := json.Unmarshal(v, &a.ClientListenerResourceNameTemplate); err != nil { + return fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + } + } + return nil +} + +// Config provides the xDS client with several key bits of information that it +// requires in its interaction with the management server. The Config is +// initialized from the bootstrap file. +type Config struct { + // XDSServer is the management server to connect to. + // + // The bootstrap file contains a list of servers (with name+creds), but we + // pick the first one. + XDSServer *ServerConfig + // CertProviderConfigs contains a mapping from certificate provider plugin + // instance names to parsed buildable configs. + CertProviderConfigs map[string]*certprovider.BuildableConfig + // ServerListenerResourceNameTemplate is a template for the name of the + // Listener resource to subscribe to for a gRPC server. + // + // If starts with "xdstp:", will be interpreted as a new-style name, + // in which case the authority of the URI will be used to select the + // relevant configuration in the "authorities" map. + // + // The token "%s", if present in this string, will be replaced with the IP + // and port on which the server is listening. (e.g., "0.0.0.0:8080", + // "[::]:8080"). For example, a value of "example/resource/%s" could become + // "example/resource/0.0.0.0:8080". If the template starts with "xdstp:", + // the replaced string will be %-encoded. + // + // There is no default; if unset, xDS-based server creation fails. + ServerListenerResourceNameTemplate string + // A template for the name of the Listener resource to subscribe to + // for a gRPC client channel. Used only when the channel is created + // with an "xds:" URI with no authority. + // + // If starts with "xdstp:", will be interpreted as a new-style name, + // in which case the authority of the URI will be used to select the + // relevant configuration in the "authorities" map. + // + // The token "%s", if present in this string, will be replaced with + // the service authority (i.e., the path part of the target URI + // used to create the gRPC channel). If the template starts with + // "xdstp:", the replaced string will be %-encoded. + // + // Defaults to "%s". + ClientDefaultListenerResourceNameTemplate string + + // Authorities is a map of authority name to corresponding configuration. + // + // This is used in the following cases: + // - A gRPC client channel is created using an "xds:" URI that includes + // an authority. + // - A gRPC client channel is created using an "xds:" URI with no + // authority, but the "client_default_listener_resource_name_template" + // field above turns it into an "xdstp:" URI. + // - A gRPC server is created and the + // "server_listener_resource_name_template" field is an "xdstp:" URI. + // + // In any of those cases, it is an error if the specified authority is + // not present in this map. + Authorities map[string]*Authority +} + +type channelCreds struct { + Type string `json:"type"` + Config json.RawMessage `json:"config"` +} + +type xdsServer struct { + ServerURI string `json:"server_uri"` + ChannelCreds []channelCreds `json:"channel_creds"` + ServerFeatures []string `json:"server_features"` +} + +func bootstrapConfigFromEnvVariable() ([]byte, error) { + fName := envconfig.XDSBootstrapFileName + fContent := envconfig.XDSBootstrapFileContent + + // Bootstrap file name has higher priority than bootstrap content. + if fName != "" { + // If file name is set + // - If file not found (or other errors), fail + // - Otherwise, use the content. + // + // Note that even if the content is invalid, we don't failover to the + // file content env variable. + logger.Debugf("xds: using bootstrap file with name %q", fName) + return bootstrapFileReadFunc(fName) + } + + if fContent != "" { + return []byte(fContent), nil + } + + return nil, fmt.Errorf("none of the bootstrap environment variables (%q or %q) defined", + envconfig.XDSBootstrapFileNameEnv, envconfig.XDSBootstrapFileContentEnv) +} + +// NewConfig returns a new instance of Config initialized by reading the +// bootstrap file found at ${GRPC_XDS_BOOTSTRAP}. +// +// Currently, we support exactly one type of credential, which is +// "google_default", where we use the host's default certs for transport +// credentials and a Google oauth token for call credentials. +// +// This function tries to process as much of the bootstrap file as possible (in +// the presence of the errors) and may return a Config object with certain +// fields left unspecified, in which case the caller should use some sane +// defaults. +func NewConfig() (*Config, error) { + // Examples of the bootstrap json can be found in the generator tests + // https://github.com/GoogleCloudPlatform/traffic-director-grpc-bootstrap/blob/master/main_test.go. + data, err := bootstrapConfigFromEnvVariable() + if err != nil { + return nil, fmt.Errorf("xds: Failed to read bootstrap config: %v", err) + } + logger.Debugf("Bootstrap content: %s", data) + return NewConfigFromContents(data) +} + +// NewConfigFromContents returns a new Config using the specified bootstrap +// file contents instead of reading the environment variable. This is only +// suitable for testing purposes. +func NewConfigFromContents(data []byte) (*Config, error) { + config := &Config{} + + var jsonData map[string]json.RawMessage + if err := json.Unmarshal(data, &jsonData); err != nil { + return nil, fmt.Errorf("xds: Failed to parse bootstrap config: %v", err) + } + + var node *v3corepb.Node + m := jsonpb.Unmarshaler{AllowUnknownFields: true} + for k, v := range jsonData { + switch k { + case "node": + // We unconditionally convert the JSON into a v3.Node proto. The v3 + // proto does not contain the deprecated field "build_version" from + // the v2 proto. We do not expect the bootstrap file to contain the + // "build_version" field. In any case, the unmarshal will succeed + // because we have set the `AllowUnknownFields` option on the + // unmarshaler. + node = &v3corepb.Node{} + if err := m.Unmarshal(bytes.NewReader(v), node); err != nil { + return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "xds_servers": + if err := json.Unmarshal(v, &config.XDSServer); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "certificate_providers": + var providerInstances map[string]json.RawMessage + if err := json.Unmarshal(v, &providerInstances); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + configs := make(map[string]*certprovider.BuildableConfig) + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + for instance, data := range providerInstances { + var nameAndConfig struct { + PluginName string `json:"plugin_name"` + Config json.RawMessage `json:"config"` + } + if err := json.Unmarshal(data, &nameAndConfig); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err) + } + + name := nameAndConfig.PluginName + parser := getBuilder(nameAndConfig.PluginName) + if parser == nil { + // We ignore plugins that we do not know about. + continue + } + bc, err := parser.ParseConfig(nameAndConfig.Config) + if err != nil { + return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err) + } + configs[instance] = bc + } + config.CertProviderConfigs = configs + case "server_listener_resource_name_template": + if err := json.Unmarshal(v, &config.ServerListenerResourceNameTemplate); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "client_default_listener_resource_name_template": + if !envconfig.XDSFederation { + logger.Warningf("xds: bootstrap field %v is not support when Federation is disabled", k) + continue + } + if err := json.Unmarshal(v, &config.ClientDefaultListenerResourceNameTemplate); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + case "authorities": + if !envconfig.XDSFederation { + logger.Warningf("xds: bootstrap field %v is not support when Federation is disabled", k) + continue + } + if err := json.Unmarshal(v, &config.Authorities); err != nil { + return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) + } + default: + logger.Warningf("Bootstrap content has unknown field: %s", k) + } + // Do not fail the xDS bootstrap when an unknown field is seen. This can + // happen when an older version client reads a newer version bootstrap + // file with new fields. + } + + if config.ClientDefaultListenerResourceNameTemplate == "" { + // Default value of the default client listener name template is "%s". + config.ClientDefaultListenerResourceNameTemplate = "%s" + } + if config.XDSServer == nil { + return nil, fmt.Errorf("xds: Required field %q not found in bootstrap %s", "xds_servers", jsonData["xds_servers"]) + } + if config.XDSServer.ServerURI == "" { + return nil, fmt.Errorf("xds: Required field %q not found in bootstrap %s", "xds_servers.server_uri", jsonData["xds_servers"]) + } + if config.XDSServer.Creds == nil { + return nil, fmt.Errorf("xds: Required field %q doesn't contain valid value in bootstrap %s", "xds_servers.channel_creds", jsonData["xds_servers"]) + } + // Post-process the authorities' client listener resource template field: + // - if set, it must start with "xdstp:///" + // - if not set, it defaults to "xdstp:///envoy.config.listener.v3.Listener/%s" + for name, authority := range config.Authorities { + prefix := fmt.Sprintf("xdstp://%s", name) + if authority.ClientListenerResourceNameTemplate == "" { + authority.ClientListenerResourceNameTemplate = prefix + "/envoy.config.listener.v3.Listener/%s" + continue + } + if !strings.HasPrefix(authority.ClientListenerResourceNameTemplate, prefix) { + return nil, fmt.Errorf("xds: field ClientListenerResourceNameTemplate %q of authority %q doesn't start with prefix %q", authority.ClientListenerResourceNameTemplate, name, prefix) + } + } + + if err := config.updateNodeProto(node); err != nil { + return nil, err + } + logger.Infof("Bootstrap config for creating xds-client: %v", pretty.ToJSON(config)) + return config, nil +} + +// updateNodeProto updates the node proto read from the bootstrap file. +// +// The input node is a v3.Node protobuf message corresponding to the JSON +// contents found in the bootstrap file. This method performs some post +// processing on it: +// 1. If the node is nil, we create an empty one here. That way, callers of this +// function can always expect that the NodeProto field is non-nil. +// 2. Some additional fields which are not expected to be set in the bootstrap +// file are populated here. +// 3. For each server config (both top level and in each authority), we set its +// node field to the v3.Node, or a v2.Node with the same content, depending on +// the server's transprot API version. +func (c *Config) updateNodeProto(node *v3corepb.Node) error { + v3 := node + if v3 == nil { + v3 = &v3corepb.Node{} + } + v3.UserAgentName = gRPCUserAgentName + v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version} + v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning) + + v2 := &v2corepb.Node{} + v3bytes, err := proto.Marshal(v3) + if err != nil { + return fmt.Errorf("xds: proto.Marshal(%v): %v", v3, err) + } + if err := proto.Unmarshal(v3bytes, v2); err != nil { + return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3bytes, err) + } + // BuildVersion is deprecated, and is replaced by user_agent_name and + // user_agent_version. But the management servers are still using the old + // field, so we will keep both set. + v2.BuildVersion = gRPCVersion + v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version} + + switch c.XDSServer.TransportAPI { + case version.TransportV2: + c.XDSServer.NodeProto = v2 + case version.TransportV3: + c.XDSServer.NodeProto = v3 + } + + for _, a := range c.Authorities { + if a.XDSServer == nil { + continue + } + switch a.XDSServer.TransportAPI { + case version.TransportV2: + a.XDSServer.NodeProto = v2 + case version.TransportV3: + a.XDSServer.NodeProto = v3 + } + } + + return nil +} diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go new file mode 100644 index 0000000000..698d4bc0bb --- /dev/null +++ b/xds/client/bootstrap/bootstrap_test.go @@ -0,0 +1,992 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + + "dubbo.apache.org/dubbo-go/v3/xds" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/google" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/credentials/tls/certprovider" + + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + structpb "github.com/golang/protobuf/ptypes/struct" +) + +var ( + v2BootstrapFileMap = map[string]string{ + "emptyNodeProto": ` + { + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownTopLevelFieldInFile": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }], + "unknownField": "foobar" + }`, + "unknownFieldInNodeProto": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "unknownField": "foobar", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ] + }] + }`, + "unknownFieldInXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "insecure" } + ], + "unknownField": "foobar" + }] + }`, + "multipleChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" }, + { "type": "google_default" } + ] + }] + }`, + "goodBootstrap": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }] + }`, + "multipleXDSServers": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [ + { + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [{ "type": "google_default" }] + }, + { + "server_uri": "backup.never.use.com:1234", + "channel_creds": [{ "type": "not-google-default" }] + } + ] + }`, + } + v3BootstrapFileMap = map[string]string{ + "serverDoesNotSupportsV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar"] + }] + }`, + "serverSupportsV3": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"] + }] + }`, + } + metadata = &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "TRAFFICDIRECTOR_GRPC_HOSTNAME": { + Kind: &structpb.Value_StringValue{StringValue: "trafficdirector"}, + }, + }, + } + v2NodeProto = &v2corepb.Node{ + Id: "ENVOY_NODE_ID", + Metadata: metadata, + BuildVersion: gRPCVersion, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + v3NodeProto = &v3corepb.Node{ + Id: "ENVOY_NODE_ID", + Metadata: metadata, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + nilCredsConfigV2 = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + CredsType: "insecure", + NodeProto: v2NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + nonNilCredsConfigV2 = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + NodeProto: v2NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + nonNilCredsConfigV3 = &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } +) + +func (c *Config) compare(want *Config) error { + if diff := cmp.Diff(c, want, + cmpopts.EquateEmpty(), + cmp.AllowUnexported(ServerConfig{}), + cmp.Comparer(proto.Equal), + cmp.Comparer(func(a, b grpc.DialOption) bool { return (a != nil) == (b != nil) }), + cmp.Transformer("certproviderconfigstring", func(a *certprovider.BuildableConfig) string { return a.String() }), + ); diff != "" { + return fmt.Errorf("diff: %v", diff) + } + return nil +} + +func fileReadFromFileMap(bootstrapFileMap map[string]string, name string) ([]byte, error) { + if b, ok := bootstrapFileMap[name]; ok { + return []byte(b), nil + } + return nil, os.ErrNotExist +} + +func setupBootstrapOverride(bootstrapFileMap map[string]string) func() { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(bootstrapFileMap, filename) + } + return func() { bootstrapFileReadFunc = oldFileReadFunc } +} + +// TODO: enable leak check for this package when +// https://github.com/googleapis/google-cloud-go/issues/2417 is fixed. + +// This function overrides the bootstrap file NAME env variable, to test the +// code that reads file with the given fileName. +func testNewConfigWithFileNameEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + t.Helper() + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = fileName + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + c, err := NewConfig() + if (err != nil) != wantError { + t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if err := c.compare(wantConfig); err != nil { + t.Fatal(err) + } +} + +// This function overrides the bootstrap file CONTENT env variable, to test the +// code that uses the content from env directly. +func testNewConfigWithFileContentEnv(t *testing.T, fileName string, wantError bool, wantConfig *Config) { + t.Helper() + b, err := bootstrapFileReadFunc(fileName) + if err != nil { + t.Skip(err) + } + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = string(b) + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + c, err := NewConfig() + if (err != nil) != wantError { + t.Fatalf("NewConfig() returned error %v, wantError: %v", err, wantError) + } + if wantError { + return + } + if err := c.compare(wantConfig); err != nil { + t.Fatal(err) + } +} + +// TestNewConfigV2ProtoFailure exercises the functionality in NewConfig with +// different bootstrap file contents which are expected to fail. +func TestNewConfigV2ProtoFailure(t *testing.T) { + bootstrapFileMap := map[string]string{ + "empty": "", + "badJSON": `["test": 123]`, + "noBalancerName": `{"node": {"id": "ENVOY_NODE_ID"}}`, + "emptyXdsServer": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + } + }`, + "emptyChannelCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }] + }`, + "nonGoogleDefaultCreds": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "not-google-default" } + ] + }] + }`, + } + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantError bool + }{ + {"nonExistentBootstrapFile", true}, + {"empty", true}, + {"badJSON", true}, + {"noBalancerName", true}, + {"emptyXdsServer", true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, true, nil) + testNewConfigWithFileContentEnv(t, test.name, true, nil) + }) + } +} + +// TestNewConfigV2ProtoSuccess exercises the functionality in NewConfig with +// different bootstrap file contents. It overrides the fileReadFunc by returning +// bootstrap file contents defined in this test, instead of reading from a file. +func TestNewConfigV2ProtoSuccess(t *testing.T) { + cancel := setupBootstrapOverride(v2BootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + }{ + { + "emptyNodeProto", &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + CredsType: "insecure", + NodeProto: &v2corepb.Node{ + BuildVersion: gRPCVersion, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + }, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + {"unknownTopLevelFieldInFile", nilCredsConfigV2}, + {"unknownFieldInNodeProto", nilCredsConfigV2}, + {"unknownFieldInXdsServer", nilCredsConfigV2}, + {"multipleChannelCreds", nonNilCredsConfigV2}, + {"goodBootstrap", nonNilCredsConfigV2}, + {"multipleXDSServers", nonNilCredsConfigV2}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig) + }) + } +} + +// TestNewConfigV3Support verifies bootstrap functionality involving support for +// the xDS v3 transport protocol. Here the client ends up using v2 or v3 based +// on what the server supports. +func TestNewConfigV3Support(t *testing.T) { + cancel := setupBootstrapOverride(v3BootstrapFileMap) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + }{ + {"serverDoesNotSupportsV3", nonNilCredsConfigV2}, + {"serverSupportsV3", nonNilCredsConfigV3}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, false, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, false, test.wantConfig) + }) + } +} + +// TestNewConfigBootstrapEnvPriority tests that the two env variables are read +// in correct priority. +// +// the case where the bootstrap file +// environment variable is not set. +func TestNewConfigBootstrapEnvPriority(t *testing.T) { + oldFileReadFunc := bootstrapFileReadFunc + bootstrapFileReadFunc = func(filename string) ([]byte, error) { + return fileReadFromFileMap(v2BootstrapFileMap, filename) + } + defer func() { bootstrapFileReadFunc = oldFileReadFunc }() + + goodFileName1 := "goodBootstrap" + goodConfig1 := nonNilCredsConfigV2 + + goodFileName2 := "serverSupportsV3" + goodFileContent2 := v3BootstrapFileMap[goodFileName2] + goodConfig2 := nonNilCredsConfigV3 + + origBootstrapFileName := envconfig.XDSBootstrapFileName + envconfig.XDSBootstrapFileName = "" + defer func() { envconfig.XDSBootstrapFileName = origBootstrapFileName }() + + origBootstrapContent := envconfig.XDSBootstrapFileContent + envconfig.XDSBootstrapFileContent = "" + defer func() { envconfig.XDSBootstrapFileContent = origBootstrapContent }() + + // When both env variables are empty, NewConfig should fail. + if _, err := NewConfig(); err == nil { + t.Errorf("NewConfig() returned nil error, expected to fail") + } + + // When one of them is set, it should be used. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = "" + if c, err := NewConfig(); err != nil || c.compare(goodConfig1) != nil { + t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil) + } + + envconfig.XDSBootstrapFileName = "" + envconfig.XDSBootstrapFileContent = goodFileContent2 + if c, err := NewConfig(); err != nil || c.compare(goodConfig2) != nil { + t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil) + } + + // Set both, file name should be read. + envconfig.XDSBootstrapFileName = goodFileName1 + envconfig.XDSBootstrapFileContent = goodFileContent2 + if c, err := NewConfig(); err != nil || c.compare(goodConfig1) != nil { + t.Errorf("NewConfig() = %v, %v, want: %v, %v", c, err, goodConfig1, nil) + } +} + +func init() { + certprovider.Register(&fakeCertProviderBuilder{}) +} + +const fakeCertProviderName = "fake-certificate-provider" + +// fakeCertProviderBuilder builds new instances of fakeCertProvider and +// interprets the config provided to it as JSON with a single key and value. +type fakeCertProviderBuilder struct{} + +// ParseConfig expects input in JSON format containing a map from string to +// string, with a single entry and mapKey being "configKey". +func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (*certprovider.BuildableConfig, error) { + config, ok := cfg.(json.RawMessage) + if !ok { + return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config) + } + var cfgData map[string]string + if err := json.Unmarshal(config, &cfgData); err != nil { + return nil, fmt.Errorf("fakeCertProviderBuilder config parsing failed: %v", err) + } + if len(cfgData) != 1 || cfgData["configKey"] == "" { + return nil, errors.New("fakeCertProviderBuilder received invalid config") + } + fc := &fakeStableConfig{config: cfgData} + return certprovider.NewBuildableConfig(fakeCertProviderName, fc.canonical(), func(certprovider.BuildOptions) certprovider.Provider { + return &fakeCertProvider{} + }), nil +} + +func (b *fakeCertProviderBuilder) Name() string { + return fakeCertProviderName +} + +type fakeStableConfig struct { + config map[string]string +} + +func (c *fakeStableConfig) canonical() []byte { + var cfg string + for k, v := range c.config { + cfg = fmt.Sprintf("%s:%s", k, v) + } + return []byte(cfg) +} + +// fakeCertProvider is an empty implementation of the Provider interface. +type fakeCertProvider struct { + certprovider.Provider +} + +func TestNewConfigWithCertificateProviders(t *testing.T) { + bootstrapFileMap := map[string]string{ + "badJSONCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"], + }], + "certificate_providers": "bad JSON" + }`, + "allUnknownCertProviders": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance1": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "unknownProviderInstance2": { + "plugin_name": "bar", + "config": {"foo": "bar"} + } + } + }`, + "badCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"], + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstanceBad": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": 666} + } + } + }`, + "goodCertProviderConfig": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ], + "server_features" : ["foo", "bar", "xds_v3"] + }], + "certificate_providers": { + "unknownProviderInstance": { + "plugin_name": "foo", + "config": {"foo": "bar"} + }, + "fakeProviderInstance": { + "plugin_name": "fake-certificate-provider", + "config": {"configKey": "configValue"} + } + } + }`, + } + + getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + parser := getBuilder(fakeCertProviderName) + if parser == nil { + t.Fatalf("missing certprovider plugin %q", fakeCertProviderName) + } + wantCfg, err := parser.ParseConfig(json.RawMessage(`{"configKey": "configValue"}`)) + if err != nil { + t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err) + } + + cancel := setupBootstrapOverride(bootstrapFileMap) + defer cancel() + + goodConfig := &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + CertProviderConfigs: map[string]*certprovider.BuildableConfig{ + "fakeProviderInstance": wantCfg, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badJSONCertProviderConfig", + wantErr: true, + }, + { + + name: "badCertProviderConfig", + wantErr: true, + }, + { + + name: "allUnknownCertProviders", + wantConfig: nonNilCredsConfigV3, + }, + { + name: "goodCertProviderConfig", + wantConfig: goodConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func TestNewConfigWithServerListenerResourceNameTemplate(t *testing.T) { + cancel := setupBootstrapOverride(map[string]string{ + "badServerListenerResourceNameTemplate:": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": 123456789 + }`, + "goodServerListenerResourceNameTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ + { "type": "google_default" } + ] + }], + "server_listener_resource_name_template": "grpc/server?xds.resource.listening_address=%s" + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badServerListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "goodServerListenerResourceNameTemplate", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV2, + NodeProto: v2NodeProto, + }, + ServerListenerResourceNameTemplate: "grpc/server?xds.resource.listening_address=%s", + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} + +func TestNewConfigWithFederation(t *testing.T) { + cancel := setupBootstrapOverride(map[string]string{ + "badClientListenerResourceNameTemplate": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443" + }], + "client_default_listener_resource_name_template": 123456789 + }`, + "badClientListenerResourceNameTemplatePerAuthority": ` + { + "node": { "id": "ENVOY_NODE_ID" }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "some/template/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["foo", "bar", "xds_v3"] + }] + } + } + }`, + "good": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "server_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + "xds_servers": [{ + "server_uri": "td.com", + "channel_creds": [ { "type": "google_default" } ], + "server_features" : ["foo", "bar", "xds_v3"] + }] + } + } + }`, + // If client_default_listener_resource_name_template is not set, it + // defaults to "%s". + "goodWithDefaultDefaultClientListenerTemplate": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }] + }`, + // If client_listener_resource_name_template in authority is not set, it + // defaults to + // "xdstp:///envoy.config.listener.v3.Listener/%s". + "goodWithDefaultClientListenerTemplatePerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { } + } + }`, + // It's OK for an authority to not have servers. The top-level server + // will be used. + "goodWithNoServerPerAuthority": ` + { + "node": { + "id": "ENVOY_NODE_ID", + "metadata": { + "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector" + } + }, + "xds_servers" : [{ + "server_uri": "trafficdirector.googleapis.com:443", + "channel_creds": [ { "type": "google_default" } ] + }], + "client_default_listener_resource_name_template": "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + "authorities": { + "xds.td.com": { + "client_listener_resource_name_template": "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s" + } + } + }`, + }) + defer cancel() + + tests := []struct { + name string + wantConfig *Config + wantErr bool + }{ + { + name: "badClientListenerResourceNameTemplate", + wantErr: true, + }, + { + name: "badClientListenerResourceNameTemplatePerAuthority", + wantErr: true, + }, + { + name: "good", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV2, + NodeProto: v2NodeProto, + }, + ServerListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/grpc/server?listening_address=%s", + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + XDSServer: &ServerConfig{ + ServerURI: "td.com", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + }, + }, + }, + }, + { + name: "goodWithDefaultDefaultClientListenerTemplate", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV2, + NodeProto: v2NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + }, + }, + { + name: "goodWithDefaultClientListenerTemplatePerAuthority", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV2, + NodeProto: v2NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + { + name: "goodWithNoServerPerAuthority", + wantConfig: &Config{ + XDSServer: &ServerConfig{ + ServerURI: "trafficdirector.googleapis.com:443", + Creds: grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()), + CredsType: "google_default", + TransportAPI: version.TransportV2, + NodeProto: v2NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "xdstp://xds.example.com/envoy.config.listener.v3.Listener/%s", + Authorities: map[string]*Authority{ + "xds.td.com": { + ClientListenerResourceNameTemplate: "xdstp://xds.td.com/envoy.config.listener.v3.Listener/%s", + }, + }, + }, + }, + } + + oldFederationSupport := envconfig.XDSFederation + envconfig.XDSFederation = true + defer func() { envconfig.XDSFederation = oldFederationSupport }() + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) + testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) + }) + } +} diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go new file mode 100644 index 0000000000..47bdbb3284 --- /dev/null +++ b/xds/client/bootstrap/logging.go @@ -0,0 +1,28 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package bootstrap + +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[xds-bootstrap] " + +var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/xds/client/bootstrap/template.go b/xds/client/bootstrap/template.go new file mode 100644 index 0000000000..9b51fcc839 --- /dev/null +++ b/xds/client/bootstrap/template.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import ( + "net/url" + "strings" +) + +// PopulateResourceTemplate populates the given template using the target +// string. "%s", if exists in the template, will be replaced with target. +// +// If the template starts with "xdstp:", the replaced string will be %-encoded. +// But note that "/" is not percent encoded. +func PopulateResourceTemplate(template, target string) string { + if !strings.Contains(template, "%s") { + return template + } + if strings.HasPrefix(template, "xdstp:") { + target = percentEncode(target) + } + return strings.Replace(template, "%s", target, -1) +} + +// percentEncode percent encode t, except for "/". See the tests for examples. +func percentEncode(t string) string { + segs := strings.Split(t, "/") + for i := range segs { + segs[i] = url.PathEscape(segs[i]) + } + return strings.Join(segs, "/") +} diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go new file mode 100644 index 0000000000..bc12eb4299 --- /dev/null +++ b/xds/client/bootstrap/template_test.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bootstrap + +import "testing" + +func Test_percentEncode(t *testing.T) { + tests := []struct { + name string + target string + want string + }{ + { + name: "normal name", + target: "server.example.com", + want: "server.example.com", + }, + { + name: "ipv4", + target: "0.0.0.0:8080", + want: "0.0.0.0:8080", + }, + { + name: "ipv6", + target: "[::1]:8080", + want: "%5B::1%5D:8080", // [ and ] are percent encoded. + }, + { + name: "/ should not be percent encoded", + target: "my/service/region", + want: "my/service/region", // "/"s are kept. + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := percentEncode(tt.target); got != tt.want { + t.Errorf("percentEncode() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestPopulateResourceTemplate(t *testing.T) { + tests := []struct { + name string + template string + target string + want string + }{ + { + name: "no %s", + template: "/name/template", + target: "[::1]:8080", + want: "/name/template", + }, + { + name: "with %s, no xdstp: prefix, ipv6", + template: "/name/template/%s", + target: "[::1]:8080", + want: "/name/template/[::1]:8080", + }, + { + name: "with %s, with xdstp: prefix", + template: "xdstp://authority.com/%s", + target: "0.0.0.0:8080", + want: "xdstp://authority.com/0.0.0.0:8080", + }, + { + name: "with %s, with xdstp: prefix, and ipv6", + template: "xdstp://authority.com/%s", + target: "[::1]:8080", + want: "xdstp://authority.com/%5B::1%5D:8080", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := PopulateResourceTemplate(tt.template, tt.target); got != tt.want { + t.Errorf("PopulateResourceTemplate() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/xds/client/client.go b/xds/client/client.go new file mode 100644 index 0000000000..1d9192378e --- /dev/null +++ b/xds/client/client.go @@ -0,0 +1,163 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// package client implements a full fledged gRPC client for the xDS API used +// by the xds resolver and balancer implementations. +package client + +import ( + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" +) + +// clientImpl is the real implementation of the xds client. The exported Client +// is a wrapper of this struct with a ref count. +// +// Implements UpdateHandler interface. +// TODO(easwars): Make a wrapper struct which implements this interface in the +// style of ccBalancerWrapper so that the Client type does not implement these +// exported methods. +type clientImpl struct { + done *grpcsync.Event + config *bootstrap.Config + + // authorityMu protects the authority fields. It's necessary because an + // authority is created when it's used. + authorityMu sync.Mutex + // authorities is a map from ServerConfig to authority. So that + // different authorities sharing the same ServerConfig can share the + // authority. + // + // The key is **ServerConfig.String()**, not the authority name. + // + // An authority is either in authorities, or idleAuthorities, + // never both. + authorities map[string]*authority + // idleAuthorities keeps the authorities that are not used (the last + // watch on it was canceled). They are kept in the cache and will be deleted + // after a timeout. The key is ServerConfig.String(). + // + // An authority is either in authorities, or idleAuthorities, + // never both. + idleAuthorities *cache.TimeoutCache + + logger *grpclog.PrefixLogger + watchExpiryTimeout time.Duration +} + +// newWithConfig returns a new xdsClient with the given config. +func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration, idleAuthorityDeleteTimeout time.Duration) (_ *clientImpl, retErr error) { + c := &clientImpl{ + done: grpcsync.NewEvent(), + config: config, + watchExpiryTimeout: watchExpiryTimeout, + + authorities: make(map[string]*authority), + idleAuthorities: cache.NewTimeoutCache(idleAuthorityDeleteTimeout), + } + + defer func() { + if retErr != nil { + c.Close() + } + }() + + c.logger = prefixLogger(c) + c.logger.Infof("Created ClientConn to xDS management server: %s", config.XDSServer) + + c.logger.Infof("Created") + return c, nil +} + +// BootstrapConfig returns the configuration read from the bootstrap file. +// Callers must treat the return value as read-only. +func (c *clientRefCounted) BootstrapConfig() *bootstrap.Config { + return c.config +} + +// Close closes the gRPC connection to the management server. +func (c *clientImpl) Close() { + if c.done.HasFired() { + return + } + c.done.Fire() + // TODO: Should we invoke the registered callbacks here with an error that + // the client is closed? + + // Note that Close needs to check for nils even if some of them are always + // set in the constructor. This is because the constructor defers Close() in + // error cases, and the fields might not be set when the error happens. + + c.authorityMu.Lock() + for _, a := range c.authorities { + a.close() + } + c.idleAuthorities.Clear(true) + c.authorityMu.Unlock() + + c.logger.Infof("Shutdown") +} + +func (c *clientImpl) filterChainUpdateValidator(fc *resource.FilterChain) error { + if fc == nil { + return nil + } + return c.securityConfigUpdateValidator(fc.SecurityCfg) +} + +func (c *clientImpl) securityConfigUpdateValidator(sc *resource.SecurityConfig) error { + if sc == nil { + return nil + } + if sc.IdentityInstanceName != "" { + if _, ok := c.config.CertProviderConfigs[sc.IdentityInstanceName]; !ok { + return fmt.Errorf("identitiy certificate provider instance name %q missing in bootstrap configuration", sc.IdentityInstanceName) + } + } + if sc.RootInstanceName != "" { + if _, ok := c.config.CertProviderConfigs[sc.RootInstanceName]; !ok { + return fmt.Errorf("root certificate provider instance name %q missing in bootstrap configuration", sc.RootInstanceName) + } + } + return nil +} + +func (c *clientImpl) updateValidator(u interface{}) error { + switch update := u.(type) { + case resource.ListenerUpdate: + if update.InboundListenerCfg == nil || update.InboundListenerCfg.FilterChains == nil { + return nil + } + return update.InboundListenerCfg.FilterChains.Validate(c.filterChainUpdateValidator) + case resource.ClusterUpdate: + return c.securityConfigUpdateValidator(update.SecurityCfg) + default: + // We currently invoke this update validation function only for LDS and + // CDS updates. In the future, if we wish to invoke it for other xDS + // updates, corresponding plumbing needs to be added to those unmarshal + // functions. + } + return nil +} diff --git a/xds/client/controller.go b/xds/client/controller.go new file mode 100644 index 0000000000..ad7b31cd4d --- /dev/null +++ b/xds/client/controller.go @@ -0,0 +1,38 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/controller" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + +type controllerInterface interface { + AddWatch(resourceType resource.ResourceType, resourceName string) + RemoveWatch(resourceType resource.ResourceType, resourceName string) + ReportLoad(server string) (*load.Store, func()) + Close() +} + +var newController = func(config *bootstrap.ServerConfig, pubsub *pubsub.Pubsub, validator resource.UpdateValidatorFunc, logger *grpclog.PrefixLogger) (controllerInterface, error) { + return controller.New(config, pubsub, validator, logger) +} diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go new file mode 100644 index 0000000000..da5362784f --- /dev/null +++ b/xds/client/controller/controller.go @@ -0,0 +1,168 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package controller contains implementation to connect to the control plane. +// Including starting the ClientConn, starting the xDS stream, and +// sending/receiving messages. +// +// All the messages are parsed by the resource package (e.g. +// UnmarshalListener()) and sent to the Pubsub watchers. +package controller + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) + +// Controller manages the connection and stream to the control plane. +// +// It keeps track of what resources are being watched, and send new requests +// when new watches are added. +// +// It takes a pubsub (as an interface) as input. When a response is received, +// it's parsed, and the updates are sent to the pubsub. +type Controller struct { + config *bootstrap.ServerConfig + updateHandler pubsub.UpdateHandler + updateValidator resource.UpdateValidatorFunc + logger *grpclog.PrefixLogger + + cc *grpc.ClientConn // Connection to the management server. + vClient version.VersionedClient + stopRunGoroutine context.CancelFunc + + backoff func(int) time.Duration + streamCh chan grpc.ClientStream + sendCh *buffer.Unbounded + + mu sync.Mutex + // Message specific watch infos, protected by the above mutex. These are + // written to, after successfully reading from the update channel, and are + // read from when recovering from a broken stream to resend the xDS + // messages. When the user of this client object cancels a watch call, + // these are set to nil. All accesses to the map protected and any value + // inside the map should be protected with the above mutex. + watchMap map[resource.ResourceType]map[string]bool + // versionMap contains the version that was acked (the version in the ack + // request that was sent on wire). The key is rType, the value is the + // version string, becaues the versions for different resource types should + // be independent. + versionMap map[resource.ResourceType]string + // nonceMap contains the nonce from the most recent received response. + nonceMap map[resource.ResourceType]string + + // Changes to map lrsClients and the lrsClient inside the map need to be + // protected by lrsMu. + // + // TODO: after LRS refactoring, each controller should only manage the LRS + // stream to its server. LRS streams to other servers should be managed by + // other controllers. + lrsMu sync.Mutex + lrsClients map[string]*lrsClient +} + +// New creates a new controller. +func New(config *bootstrap.ServerConfig, updateHandler pubsub.UpdateHandler, validator resource.UpdateValidatorFunc, logger *grpclog.PrefixLogger) (_ *Controller, retErr error) { + switch { + case config == nil: + return nil, errors.New("xds: no xds_server provided") + case config.ServerURI == "": + return nil, errors.New("xds: no xds_server name provided in options") + case config.Creds == nil: + return nil, errors.New("xds: no credentials provided in options") + case config.NodeProto == nil: + return nil, errors.New("xds: no node_proto provided in options") + } + + dopts := []grpc.DialOption{ + config.Creds, + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 5 * time.Minute, + Timeout: 20 * time.Second, + }), + } + + ret := &Controller{ + config: config, + updateValidator: validator, + updateHandler: updateHandler, + + backoff: backoff.DefaultExponential.Backoff, // TODO: should this be configurable? + streamCh: make(chan grpc.ClientStream, 1), + sendCh: buffer.NewUnbounded(), + watchMap: make(map[resource.ResourceType]map[string]bool), + versionMap: make(map[resource.ResourceType]string), + nonceMap: make(map[resource.ResourceType]string), + + lrsClients: make(map[string]*lrsClient), + } + + defer func() { + if retErr != nil { + ret.Close() + } + }() + + cc, err := grpc.Dial(config.ServerURI, dopts...) + if err != nil { + // An error from a non-blocking dial indicates something serious. + return nil, fmt.Errorf("xds: failed to dial control plane {%s}: %v", config.ServerURI, err) + } + ret.cc = cc + + builder := version.GetAPIClientBuilder(config.TransportAPI) + if builder == nil { + return nil, fmt.Errorf("no client builder for xDS API version: %v", config.TransportAPI) + } + apiClient, err := builder(version.BuildOptions{NodeProto: config.NodeProto, Logger: logger}) + if err != nil { + return nil, err + } + ret.vClient = apiClient + + ctx, cancel := context.WithCancel(context.Background()) + ret.stopRunGoroutine = cancel + go ret.run(ctx) + + return ret, nil +} + +// Close closes the controller. +func (t *Controller) Close() { + // Note that Close needs to check for nils even if some of them are always + // set in the constructor. This is because the constructor defers Close() in + // error cases, and the fields might not be set when the error happens. + if t.stopRunGoroutine != nil { + t.stopRunGoroutine() + } + if t.cc != nil { + t.cc.Close() + } +} diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go new file mode 100644 index 0000000000..5408cb123b --- /dev/null +++ b/xds/client/controller/loadreport.go @@ -0,0 +1,144 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + + "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "google.golang.org/grpc" +) + +// ReportLoad starts an load reporting stream to the given server. If the server +// is not an empty string, and is different from the management server, a new +// ClientConn will be created. +// +// The same options used for creating the Client will be used (including +// NodeProto, and dial options if necessary). +// +// It returns a Store for the user to report loads, a function to cancel the +// load reporting stream. +// +// TODO: LRS refactor; maybe a new controller should be created for a separate +// server, so that the same stream can be shared by different reporters to the +// same server, even if they originate from different Controllers. +func (c *Controller) ReportLoad(server string) (*load.Store, func()) { + c.lrsMu.Lock() + defer c.lrsMu.Unlock() + + // If there's already a client to this server, use it. Otherwise, create + // one. + lrsC, ok := c.lrsClients[server] + if !ok { + lrsC = newLRSClient(c, server) + c.lrsClients[server] = lrsC + } + + store := lrsC.ref() + return store, func() { + // This is a callback, need to hold lrsMu. + c.lrsMu.Lock() + defer c.lrsMu.Unlock() + if lrsC.unRef() { + // Delete the lrsClient from map if this is the last reference. + delete(c.lrsClients, server) + } + } +} + +// lrsClient maps to one lrsServer. It contains: +// - a ClientConn to this server (only if it's different from the management +// server) +// - a load.Store that contains loads only for this server +type lrsClient struct { + parent *Controller + server string + + cc *grpc.ClientConn // nil if the server is same as the management server + refCount int + cancelStream func() + loadStore *load.Store +} + +// newLRSClient creates a new LRS stream to the server. +func newLRSClient(parent *Controller, server string) *lrsClient { + return &lrsClient{ + parent: parent, + server: server, + refCount: 0, + } +} + +// ref increments the refCount. If this is the first ref, it starts the LRS stream. +// +// Not thread-safe, caller needs to synchronize. +func (lrsC *lrsClient) ref() *load.Store { + lrsC.refCount++ + if lrsC.refCount == 1 { + lrsC.startStream() + } + return lrsC.loadStore +} + +// unRef decrements the refCount, and closes the stream if refCount reaches 0 +// (and close the cc if cc is not xDS cc). It returns whether refCount reached 0 +// after this call. +// +// Not thread-safe, caller needs to synchronize. +func (lrsC *lrsClient) unRef() (closed bool) { + lrsC.refCount-- + if lrsC.refCount != 0 { + return false + } + lrsC.parent.logger.Infof("Stopping load report to server: %s", lrsC.server) + lrsC.cancelStream() + if lrsC.cc != nil { + lrsC.cc.Close() + } + return true +} + +// startStream starts the LRS stream to the server. If server is not the same +// management server from the parent, it also creates a ClientConn. +func (lrsC *lrsClient) startStream() { + var cc *grpc.ClientConn + + lrsC.parent.logger.Infof("Starting load report to server: %s", lrsC.server) + if lrsC.server == "" || lrsC.server == lrsC.parent.config.ServerURI { + // Reuse the xDS client if server is the same. + cc = lrsC.parent.cc + } else { + lrsC.parent.logger.Infof("LRS server is different from management server, starting a new ClientConn") + ccNew, err := grpc.Dial(lrsC.server, lrsC.parent.config.Creds) + if err != nil { + // An error from a non-blocking dial indicates something serious. + lrsC.parent.logger.Infof("xds: failed to dial load report server {%s}: %v", lrsC.server, err) + return + } + cc = ccNew + lrsC.cc = ccNew + } + + var ctx context.Context + ctx, lrsC.cancelStream = context.WithCancel(context.Background()) + + // Create the store and stream. + lrsC.loadStore = load.NewStore() + go lrsC.parent.reportLoad(ctx, cc, version.LoadReportingOptions{LoadStore: lrsC.loadStore}) +} diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go new file mode 100644 index 0000000000..0caed731e5 --- /dev/null +++ b/xds/client/controller/transport.go @@ -0,0 +1,429 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package controller + +import ( + "context" + "fmt" + "time" + + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" +) + +// AddWatch adds a watch for an xDS resource given its type and name. +func (t *Controller) AddWatch(rType resource.ResourceType, resourceName string) { + t.sendCh.Put(&watchAction{ + rType: rType, + remove: false, + resource: resourceName, + }) +} + +// RemoveWatch cancels an already registered watch for an xDS resource +// given its type and name. +func (t *Controller) RemoveWatch(rType resource.ResourceType, resourceName string) { + t.sendCh.Put(&watchAction{ + rType: rType, + remove: true, + resource: resourceName, + }) +} + +// run starts an ADS stream (and backs off exponentially, if the previous +// stream failed without receiving a single reply) and runs the sender and +// receiver routines to send and receive data from the stream respectively. +func (t *Controller) run(ctx context.Context) { + go t.send(ctx) + // TODO: start a goroutine monitoring ClientConn's connectivity state, and + // report error (and log) when stats is transient failure. + + retries := 0 + for { + select { + case <-ctx.Done(): + return + default: + } + + if retries != 0 { + timer := time.NewTimer(t.backoff(retries)) + select { + case <-timer.C: + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + } + } + + retries++ + stream, err := t.vClient.NewStream(ctx, t.cc) + if err != nil { + t.updateHandler.NewConnectionError(err) + t.logger.Warningf("xds: ADS stream creation failed: %v", err) + continue + } + t.logger.Infof("ADS stream created") + + select { + case <-t.streamCh: + default: + } + t.streamCh <- stream + if t.recv(stream) { + retries = 0 + } + } +} + +// send is a separate goroutine for sending watch requests on the xds stream. +// +// It watches the stream channel for new streams, and the request channel for +// new requests to send on the stream. +// +// For each new request (watchAction), it's +// - processed and added to the watch map +// - so resend will pick them up when there are new streams +// - sent on the current stream if there's one +// - the current stream is cleared when any send on it fails +// +// For each new stream, all the existing requests will be resent. +// +// Note that this goroutine doesn't do anything to the old stream when there's a +// new one. In fact, there should be only one stream in progress, and new one +// should only be created when the old one fails (recv returns an error). +func (t *Controller) send(ctx context.Context) { + var stream grpc.ClientStream + for { + select { + case <-ctx.Done(): + return + case stream = <-t.streamCh: + if !t.sendExisting(stream) { + // send failed, clear the current stream. + stream = nil + } + case u := <-t.sendCh.Get(): + t.sendCh.Load() + + var ( + target []string + rType resource.ResourceType + version, nonce, errMsg string + send bool + ) + switch update := u.(type) { + case *watchAction: + target, rType, version, nonce = t.processWatchInfo(update) + case *ackAction: + target, rType, version, nonce, send = t.processAckInfo(update, stream) + if !send { + continue + } + errMsg = update.errMsg + } + if stream == nil { + // There's no stream yet. Skip the request. This request + // will be resent to the new streams. If no stream is + // created, the watcher will timeout (same as server not + // sending response back). + continue + } + if err := t.vClient.SendRequest(stream, target, rType, version, nonce, errMsg); err != nil { + t.logger.Warningf("ADS request for {target: %q, type: %v, version: %q, nonce: %q} failed: %v", target, rType, version, nonce, err) + // send failed, clear the current stream. + stream = nil + } + } + } +} + +// sendExisting sends out xDS requests for registered watchers when recovering +// from a broken stream. +// +// We call stream.Send() here with the lock being held. It should be OK to do +// that here because the stream has just started and Send() usually returns +// quickly (once it pushes the message onto the transport layer) and is only +// ever blocked if we don't have enough flow control quota. +func (t *Controller) sendExisting(stream grpc.ClientStream) bool { + t.mu.Lock() + defer t.mu.Unlock() + + // Reset the ack versions when the stream restarts. + t.versionMap = make(map[resource.ResourceType]string) + t.nonceMap = make(map[resource.ResourceType]string) + + for rType, s := range t.watchMap { + if err := t.vClient.SendRequest(stream, mapToSlice(s), rType, "", "", ""); err != nil { + t.logger.Warningf("ADS request failed: %v", err) + return false + } + } + + return true +} + +// recv receives xDS responses on the provided ADS stream and branches out to +// message specific handlers. +func (t *Controller) recv(stream grpc.ClientStream) bool { + success := false + for { + resp, err := t.vClient.RecvResponse(stream) + if err != nil { + t.updateHandler.NewConnectionError(err) + t.logger.Warningf("ADS stream is closed with error: %v", err) + return success + } + + rType, version, nonce, err := t.handleResponse(resp) + + if e, ok := err.(resourceversion.ErrResourceTypeUnsupported); ok { + t.logger.Warningf("%s", e.ErrStr) + continue + } + if err != nil { + t.sendCh.Put(&ackAction{ + rType: rType, + version: "", + nonce: nonce, + errMsg: err.Error(), + stream: stream, + }) + t.logger.Warningf("Sending NACK for response type: %v, version: %v, nonce: %v, reason: %v", rType, version, nonce, err) + continue + } + t.sendCh.Put(&ackAction{ + rType: rType, + version: version, + nonce: nonce, + stream: stream, + }) + t.logger.Infof("Sending ACK for response type: %v, version: %v, nonce: %v", rType, version, nonce) + success = true + } +} + +func (t *Controller) handleResponse(resp proto.Message) (resource.ResourceType, string, string, error) { + rType, resources, version, nonce, err := t.vClient.ParseResponse(resp) + if err != nil { + return rType, version, nonce, err + } + opts := &resource.UnmarshalOptions{ + Version: version, + Resources: resources, + Logger: t.logger, + UpdateValidator: t.updateValidator, + } + var md resource.UpdateMetadata + switch rType { + case resource.ListenerResource: + var update map[string]resource.ListenerUpdateErrTuple + update, md, err = resource.UnmarshalListener(opts) + t.updateHandler.NewListeners(update, md) + case resource.RouteConfigResource: + var update map[string]resource.RouteConfigUpdateErrTuple + update, md, err = resource.UnmarshalRouteConfig(opts) + t.updateHandler.NewRouteConfigs(update, md) + case resource.ClusterResource: + var update map[string]resource.ClusterUpdateErrTuple + update, md, err = resource.UnmarshalCluster(opts) + t.updateHandler.NewClusters(update, md) + case resource.EndpointsResource: + var update map[string]resource.EndpointsUpdateErrTuple + update, md, err = resource.UnmarshalEndpoints(opts) + t.updateHandler.NewEndpoints(update, md) + default: + return rType, "", "", resourceversion.ErrResourceTypeUnsupported{ + ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", rType), + } + } + return rType, version, nonce, err +} + +func mapToSlice(m map[string]bool) []string { + ret := make([]string, 0, len(m)) + for i := range m { + ret = append(ret, i) + } + return ret +} + +type watchAction struct { + rType resource.ResourceType + remove bool // Whether this is to remove watch for the resource. + resource string +} + +// processWatchInfo pulls the fields needed by the request from a watchAction. +// +// It also updates the watch map. +func (t *Controller) processWatchInfo(w *watchAction) (target []string, rType resource.ResourceType, ver, nonce string) { + t.mu.Lock() + defer t.mu.Unlock() + + var current map[string]bool + current, ok := t.watchMap[w.rType] + if !ok { + current = make(map[string]bool) + t.watchMap[w.rType] = current + } + + if w.remove { + delete(current, w.resource) + if len(current) == 0 { + delete(t.watchMap, w.rType) + } + } else { + current[w.resource] = true + } + + rType = w.rType + target = mapToSlice(current) + // We don't reset version or nonce when a new watch is started. The version + // and nonce from previous response are carried by the request unless the + // stream is recreated. + ver = t.versionMap[rType] + nonce = t.nonceMap[rType] + return target, rType, ver, nonce +} + +type ackAction struct { + rType resource.ResourceType + version string // NACK if version is an empty string. + nonce string + errMsg string // Empty unless it's a NACK. + // ACK/NACK are tagged with the stream it's for. When the stream is down, + // all the ACK/NACK for this stream will be dropped, and the version/nonce + // won't be updated. + stream grpc.ClientStream +} + +// processAckInfo pulls the fields needed by the ack request from a ackAction. +// +// If no active watch is found for this ack, it returns false for send. +func (t *Controller) processAckInfo(ack *ackAction, stream grpc.ClientStream) (target []string, rType resource.ResourceType, version, nonce string, send bool) { + if ack.stream != stream { + // If ACK's stream isn't the current sending stream, this means the ACK + // was pushed to queue before the old stream broke, and a new stream has + // been started since. Return immediately here so we don't update the + // nonce for the new stream. + return nil, resource.UnknownResource, "", "", false + } + rType = ack.rType + + t.mu.Lock() + defer t.mu.Unlock() + + // Update the nonce no matter if we are going to send the ACK request on + // wire. We may not send the request if the watch is canceled. But the nonce + // needs to be updated so the next request will have the right nonce. + nonce = ack.nonce + t.nonceMap[rType] = nonce + + s, ok := t.watchMap[rType] + if !ok || len(s) == 0 { + // We don't send the request ack if there's no active watch (this can be + // either the server sends responses before any request, or the watch is + // canceled while the ackAction is in queue), because there's no resource + // name. And if we send a request with empty resource name list, the + // server may treat it as a wild card and send us everything. + return nil, resource.UnknownResource, "", "", false + } + send = true + target = mapToSlice(s) + + version = ack.version + if version == "" { + // This is a nack, get the previous acked version. + version = t.versionMap[rType] + // version will still be an empty string if rType isn't + // found in versionMap, this can happen if there wasn't any ack + // before. + } else { + t.versionMap[rType] = version + } + return target, rType, version, nonce, send +} + +// reportLoad starts an LRS stream to report load data to the management server. +// It blocks until the context is cancelled. +func (t *Controller) reportLoad(ctx context.Context, cc *grpc.ClientConn, opts controllerversion.LoadReportingOptions) { + retries := 0 + for { + if ctx.Err() != nil { + return + } + + if retries != 0 { + timer := time.NewTimer(t.backoff(retries)) + select { + case <-timer.C: + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return + } + } + + retries++ + stream, err := t.vClient.NewLoadStatsStream(ctx, cc) + if err != nil { + t.logger.Warningf("lrs: failed to create stream: %v", err) + continue + } + t.logger.Infof("lrs: created LRS stream") + + if err := t.vClient.SendFirstLoadStatsRequest(stream); err != nil { + t.logger.Warningf("lrs: failed to send first request: %v", err) + continue + } + + clusters, interval, err := t.vClient.HandleLoadStatsResponse(stream) + if err != nil { + t.logger.Warningf("%v", err) + continue + } + + retries = 0 + t.sendLoads(ctx, stream, opts.LoadStore, clusters, interval) + } +} + +func (t *Controller) sendLoads(ctx context.Context, stream grpc.ClientStream, store *load.Store, clusterNames []string, interval time.Duration) { + tick := time.NewTicker(interval) + defer tick.Stop() + for { + select { + case <-tick.C: + case <-ctx.Done(): + return + } + if err := t.vClient.SendLoadStatsRequest(stream, store.Stats(clusterNames)); err != nil { + t.logger.Warningf("%v", err) + return + } + } +} diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go new file mode 100644 index 0000000000..df3fd610b7 --- /dev/null +++ b/xds/client/controller/version/v2/client.go @@ -0,0 +1,155 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package v2 provides xDS v2 transport protocol specific functionality. +package v2 + +import ( + "context" + "fmt" + + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" + + v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" + statuspb "google.golang.org/genproto/googleapis/rpc/status" +) + +func init() { + controllerversion.RegisterAPIClientBuilder(resourceversion.TransportV2, newClient) +} + +var ( + resourceTypeToURL = map[resource.ResourceType]string{ + resource.ListenerResource: resourceversion.V2ListenerURL, + resource.RouteConfigResource: resourceversion.V2RouteConfigURL, + resource.ClusterResource: resourceversion.V2ClusterURL, + resource.EndpointsResource: resourceversion.V2EndpointsURL, + } +) + +func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { + nodeProto, ok := opts.NodeProto.(*v2corepb.Node) + if !ok { + return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, (*v2corepb.Node)(nil)) + } + v2c := &client{nodeProto: nodeProto, logger: opts.Logger} + return v2c, nil +} + +type adsStream v2adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient + +// client performs the actual xDS RPCs using the xDS v2 API. It creates a +// single ADS stream on which the different types of xDS requests and responses +// are multiplexed. +type client struct { + nodeProto *v2corepb.Node + logger *grpclog.PrefixLogger +} + +func (v2c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { + return v2adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) +} + +// SendRequest sends out a DiscoveryRequest for the given resourceNames, of type +// rType, on the provided stream. +// +// version is the ack version to be sent with the request +// - If this is the new request (not an ack/nack), version will be empty. +// - If this is an ack, version will be the version from the response. +// - If this is a nack, version will be the previous acked version (from +// versionMap). If there was no ack before, it will be empty. +func (v2c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType resource.ResourceType, version, nonce, errMsg string) error { + stream, ok := s.(adsStream) + if !ok { + return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s) + } + req := &v2xdspb.DiscoveryRequest{ + Node: v2c.nodeProto, + TypeUrl: resourceTypeToURL[rType], + ResourceNames: resourceNames, + VersionInfo: version, + ResponseNonce: nonce, + } + if errMsg != "" { + req.ErrorDetail = &statuspb.Status{ + Code: int32(codes.InvalidArgument), Message: errMsg, + } + } + if err := stream.Send(req); err != nil { + return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err) + } + v2c.logger.Debugf("ADS request sent: %v", pretty.ToJSON(req)) + return nil +} + +// RecvResponse blocks on the receipt of one response message on the provided +// stream. +func (v2c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) { + stream, ok := s.(adsStream) + if !ok { + return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s) + } + + resp, err := stream.Recv() + if err != nil { + return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err) + } + v2c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl()) + v2c.logger.Debugf("ADS response received: %v", pretty.ToJSON(resp)) + return resp, nil +} + +func (v2c *client) ParseResponse(r proto.Message) (resource.ResourceType, []*anypb.Any, string, string, error) { + rType := resource.UnknownResource + resp, ok := r.(*v2xdspb.DiscoveryResponse) + if !ok { + return rType, nil, "", "", fmt.Errorf("xds: unsupported message type: %T", resp) + } + + // Note that the xDS transport protocol is versioned independently of + // the resource types, and it is supported to transfer older versions + // of resource types using new versions of the transport protocol, or + // vice-versa. Hence we need to handle v3 type_urls as well here. + var err error + url := resp.GetTypeUrl() + switch { + case resource.IsListenerResource(url): + rType = resource.ListenerResource + case resource.IsRouteConfigResource(url): + rType = resource.RouteConfigResource + case resource.IsClusterResource(url): + rType = resource.ClusterResource + case resource.IsEndpointsResource(url): + rType = resource.EndpointsResource + default: + return rType, nil, "", "", controllerversion.ErrResourceTypeUnsupported{ + ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()), + } + } + return rType, resp.GetResources(), resp.GetVersionInfo(), resp.GetNonce(), err +} diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go new file mode 100644 index 0000000000..69d7b5df80 --- /dev/null +++ b/xds/client/controller/version/v2/loadreport.go @@ -0,0 +1,153 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v2 + +import ( + "context" + "errors" + "fmt" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" + lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" + lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" + "google.golang.org/grpc" +) + +const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" + +type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient + +func (v2c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { + c := lrsgrpc.NewLoadReportingServiceClient(cc) + return c.StreamLoadStats(ctx) +} + +func (v2c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error { + stream, ok := s.(lrsStream) + if !ok { + return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s) + } + node := proto.Clone(v2c.nodeProto).(*v2corepb.Node) + if node == nil { + node = &v2corepb.Node{} + } + node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters) + + req := &lrspb.LoadStatsRequest{Node: node} + v2c.logger.Infof("lrs: sending init LoadStatsRequest: %v", pretty.ToJSON(req)) + return stream.Send(req) +} + +func (v2c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.Duration, error) { + stream, ok := s.(lrsStream) + if !ok { + return nil, 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s) + } + + resp, err := stream.Recv() + if err != nil { + return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err) + } + v2c.logger.Infof("lrs: received first LoadStatsResponse: %+v", pretty.ToJSON(resp)) + + interval, err := ptypes.Duration(resp.GetLoadReportingInterval()) + if err != nil { + return nil, 0, fmt.Errorf("lrs: failed to convert report interval: %v", err) + } + + if resp.ReportEndpointGranularity { + // TODO: fixme to support per endpoint loads. + return nil, 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation") + } + + clusters := resp.Clusters + if resp.SendAllClusters { + // Return nil to send stats for all clusters. + clusters = nil + } + + return clusters, interval, nil +} + +func (v2c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error { + stream, ok := s.(lrsStream) + if !ok { + return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s) + } + + clusterStats := make([]*v2endpointpb.ClusterStats, 0, len(loads)) + for _, sd := range loads { + droppedReqs := make([]*v2endpointpb.ClusterStats_DroppedRequests, 0, len(sd.Drops)) + for category, count := range sd.Drops { + droppedReqs = append(droppedReqs, &v2endpointpb.ClusterStats_DroppedRequests{ + Category: category, + DroppedCount: count, + }) + } + localityStats := make([]*v2endpointpb.UpstreamLocalityStats, 0, len(sd.LocalityStats)) + for l, localityData := range sd.LocalityStats { + lid, err := resource.LocalityIDFromString(l) + if err != nil { + return err + } + loadMetricStats := make([]*v2endpointpb.EndpointLoadMetricStats, 0, len(localityData.LoadStats)) + for name, loadData := range localityData.LoadStats { + loadMetricStats = append(loadMetricStats, &v2endpointpb.EndpointLoadMetricStats{ + MetricName: name, + NumRequestsFinishedWithMetric: loadData.Count, + TotalMetricValue: loadData.Sum, + }) + } + localityStats = append(localityStats, &v2endpointpb.UpstreamLocalityStats{ + Locality: &v2corepb.Locality{ + Region: lid.Region, + Zone: lid.Zone, + SubZone: lid.SubZone, + }, + TotalSuccessfulRequests: localityData.RequestStats.Succeeded, + TotalRequestsInProgress: localityData.RequestStats.InProgress, + TotalErrorRequests: localityData.RequestStats.Errored, + LoadMetricStats: loadMetricStats, + UpstreamEndpointStats: nil, // TODO: populate for per endpoint loads. + }) + } + + clusterStats = append(clusterStats, &v2endpointpb.ClusterStats{ + ClusterName: sd.Cluster, + ClusterServiceName: sd.Service, + UpstreamLocalityStats: localityStats, + TotalDroppedRequests: sd.TotalDrops, + DroppedRequests: droppedReqs, + LoadReportInterval: ptypes.DurationProto(sd.ReportInterval), + }) + + } + + req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats} + v2c.logger.Infof("lrs: sending LRS loads: %+v", pretty.ToJSON(req)) + return stream.Send(req) +} diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go new file mode 100644 index 0000000000..9fa804febc --- /dev/null +++ b/xds/client/controller/version/v3/client.go @@ -0,0 +1,157 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package v3 provides xDS v3 transport protocol specific functionality. +package v3 + +import ( + "context" + "fmt" + + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "github.com/golang/protobuf/proto" + statuspb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +) + +func init() { + controllerversion.RegisterAPIClientBuilder(resourceversion.TransportV3, newClient) +} + +var ( + resourceTypeToURL = map[resource.ResourceType]string{ + resource.ListenerResource: resourceversion.V3ListenerURL, + resource.RouteConfigResource: resourceversion.V3RouteConfigURL, + resource.ClusterResource: resourceversion.V3ClusterURL, + resource.EndpointsResource: resourceversion.V3EndpointsURL, + } +) + +func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { + nodeProto, ok := opts.NodeProto.(*v3corepb.Node) + if !ok { + return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v3corepb.Node{}) + } + v3c := &client{ + nodeProto: nodeProto, logger: opts.Logger, + } + return v3c, nil +} + +type adsStream v3adsgrpc.AggregatedDiscoveryService_StreamAggregatedResourcesClient + +// client performs the actual xDS RPCs using the xDS v3 API. It creates a +// single ADS stream on which the different types of xDS requests and responses +// are multiplexed. +type client struct { + nodeProto *v3corepb.Node + logger *grpclog.PrefixLogger +} + +func (v3c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { + return v3adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) +} + +// SendRequest sends out a DiscoveryRequest for the given resourceNames, of type +// rType, on the provided stream. +// +// version is the ack version to be sent with the request +// - If this is the new request (not an ack/nack), version will be empty. +// - If this is an ack, version will be the version from the response. +// - If this is a nack, version will be the previous acked version (from +// versionMap). If there was no ack before, it will be empty. +func (v3c *client) SendRequest(s grpc.ClientStream, resourceNames []string, rType resource.ResourceType, version, nonce, errMsg string) error { + stream, ok := s.(adsStream) + if !ok { + return fmt.Errorf("xds: Attempt to send request on unsupported stream type: %T", s) + } + req := &v3discoverypb.DiscoveryRequest{ + Node: v3c.nodeProto, + TypeUrl: resourceTypeToURL[rType], + ResourceNames: resourceNames, + VersionInfo: version, + ResponseNonce: nonce, + } + if errMsg != "" { + req.ErrorDetail = &statuspb.Status{ + Code: int32(codes.InvalidArgument), Message: errMsg, + } + } + if err := stream.Send(req); err != nil { + return fmt.Errorf("xds: stream.Send(%+v) failed: %v", req, err) + } + v3c.logger.Debugf("ADS request sent: %v", pretty.ToJSON(req)) + return nil +} + +// RecvResponse blocks on the receipt of one response message on the provided +// stream. +func (v3c *client) RecvResponse(s grpc.ClientStream) (proto.Message, error) { + stream, ok := s.(adsStream) + if !ok { + return nil, fmt.Errorf("xds: Attempt to receive response on unsupported stream type: %T", s) + } + + resp, err := stream.Recv() + if err != nil { + return nil, fmt.Errorf("xds: stream.Recv() failed: %v", err) + } + v3c.logger.Infof("ADS response received, type: %v", resp.GetTypeUrl()) + v3c.logger.Debugf("ADS response received: %+v", pretty.ToJSON(resp)) + return resp, nil +} + +func (v3c *client) ParseResponse(r proto.Message) (resource.ResourceType, []*anypb.Any, string, string, error) { + rType := resource.UnknownResource + resp, ok := r.(*v3discoverypb.DiscoveryResponse) + if !ok { + return rType, nil, "", "", fmt.Errorf("xds: unsupported message type: %T", resp) + } + + // Note that the xDS transport protocol is versioned independently of + // the resource types, and it is supported to transfer older versions + // of resource types using new versions of the transport protocol, or + // vice-versa. Hence we need to handle v3 type_urls as well here. + var err error + url := resp.GetTypeUrl() + switch { + case resource.IsListenerResource(url): + rType = resource.ListenerResource + case resource.IsRouteConfigResource(url): + rType = resource.RouteConfigResource + case resource.IsClusterResource(url): + rType = resource.ClusterResource + case resource.IsEndpointsResource(url): + rType = resource.EndpointsResource + default: + return rType, nil, "", "", controllerversion.ErrResourceTypeUnsupported{ + ErrStr: fmt.Sprintf("Resource type %v unknown in response from server", resp.GetTypeUrl()), + } + } + return rType, resp.GetResources(), resp.GetVersionInfo(), resp.GetNonce(), err +} diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go new file mode 100644 index 0000000000..6a55f9ab46 --- /dev/null +++ b/xds/client/controller/version/v3/loadreport.go @@ -0,0 +1,152 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package v3 + +import ( + "context" + "errors" + "fmt" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + "google.golang.org/grpc" +) + +const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" + +type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient + +func (v3c *client) NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { + c := lrsgrpc.NewLoadReportingServiceClient(cc) + return c.StreamLoadStats(ctx) +} + +func (v3c *client) SendFirstLoadStatsRequest(s grpc.ClientStream) error { + stream, ok := s.(lrsStream) + if !ok { + return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s) + } + node := proto.Clone(v3c.nodeProto).(*v3corepb.Node) + if node == nil { + node = &v3corepb.Node{} + } + node.ClientFeatures = append(node.ClientFeatures, clientFeatureLRSSendAllClusters) + + req := &lrspb.LoadStatsRequest{Node: node} + v3c.logger.Infof("lrs: sending init LoadStatsRequest: %v", pretty.ToJSON(req)) + return stream.Send(req) +} + +func (v3c *client) HandleLoadStatsResponse(s grpc.ClientStream) ([]string, time.Duration, error) { + stream, ok := s.(lrsStream) + if !ok { + return nil, 0, fmt.Errorf("lrs: Attempt to receive response on unsupported stream type: %T", s) + } + + resp, err := stream.Recv() + if err != nil { + return nil, 0, fmt.Errorf("lrs: failed to receive first response: %v", err) + } + v3c.logger.Infof("lrs: received first LoadStatsResponse: %+v", pretty.ToJSON(resp)) + + interval, err := ptypes.Duration(resp.GetLoadReportingInterval()) + if err != nil { + return nil, 0, fmt.Errorf("lrs: failed to convert report interval: %v", err) + } + + if resp.ReportEndpointGranularity { + // TODO: fixme to support per endpoint loads. + return nil, 0, errors.New("lrs: endpoint loads requested, but not supported by current implementation") + } + + clusters := resp.Clusters + if resp.SendAllClusters { + // Return nil to send stats for all clusters. + clusters = nil + } + + return clusters, interval, nil +} + +func (v3c *client) SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error { + stream, ok := s.(lrsStream) + if !ok { + return fmt.Errorf("lrs: Attempt to send request on unsupported stream type: %T", s) + } + + clusterStats := make([]*v3endpointpb.ClusterStats, 0, len(loads)) + for _, sd := range loads { + droppedReqs := make([]*v3endpointpb.ClusterStats_DroppedRequests, 0, len(sd.Drops)) + for category, count := range sd.Drops { + droppedReqs = append(droppedReqs, &v3endpointpb.ClusterStats_DroppedRequests{ + Category: category, + DroppedCount: count, + }) + } + localityStats := make([]*v3endpointpb.UpstreamLocalityStats, 0, len(sd.LocalityStats)) + for l, localityData := range sd.LocalityStats { + lid, err := resource.LocalityIDFromString(l) + if err != nil { + return err + } + loadMetricStats := make([]*v3endpointpb.EndpointLoadMetricStats, 0, len(localityData.LoadStats)) + for name, loadData := range localityData.LoadStats { + loadMetricStats = append(loadMetricStats, &v3endpointpb.EndpointLoadMetricStats{ + MetricName: name, + NumRequestsFinishedWithMetric: loadData.Count, + TotalMetricValue: loadData.Sum, + }) + } + localityStats = append(localityStats, &v3endpointpb.UpstreamLocalityStats{ + Locality: &v3corepb.Locality{ + Region: lid.Region, + Zone: lid.Zone, + SubZone: lid.SubZone, + }, + TotalSuccessfulRequests: localityData.RequestStats.Succeeded, + TotalRequestsInProgress: localityData.RequestStats.InProgress, + TotalErrorRequests: localityData.RequestStats.Errored, + LoadMetricStats: loadMetricStats, + UpstreamEndpointStats: nil, // TODO: populate for per endpoint loads. + }) + } + + clusterStats = append(clusterStats, &v3endpointpb.ClusterStats{ + ClusterName: sd.Cluster, + ClusterServiceName: sd.Service, + UpstreamLocalityStats: localityStats, + TotalDroppedRequests: sd.TotalDrops, + DroppedRequests: droppedReqs, + LoadReportInterval: ptypes.DurationProto(sd.ReportInterval), + }) + } + + req := &lrspb.LoadStatsRequest{ClusterStats: clusterStats} + v3c.logger.Infof("lrs: sending LRS loads: %+v", pretty.ToJSON(req)) + return stream.Send(req) +} diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go new file mode 100644 index 0000000000..64f4f4dc19 --- /dev/null +++ b/xds/client/controller/version/version.go @@ -0,0 +1,123 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package version defines APIs to deal with different versions of xDS. +package version + +import ( + "context" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/anypb" +) + +var ( + m = make(map[version.TransportAPI]func(opts BuildOptions) (VersionedClient, error)) +) + +// RegisterAPIClientBuilder registers a client builder for xDS transport protocol +// version specified by b.Version(). +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple builders are +// registered for the same version, the one registered last will take effect. +func RegisterAPIClientBuilder(v version.TransportAPI, f func(opts BuildOptions) (VersionedClient, error)) { + m[v] = f +} + +// GetAPIClientBuilder returns the client builder registered for the provided +// xDS transport API version. +func GetAPIClientBuilder(version version.TransportAPI) func(opts BuildOptions) (VersionedClient, error) { + if f, ok := m[version]; ok { + return f + } + return nil +} + +// BuildOptions contains options to be passed to client builders. +type BuildOptions struct { + // NodeProto contains the Node proto to be used in xDS requests. The actual + // type depends on the transport protocol version used. + NodeProto proto.Message + // // Backoff returns the amount of time to backoff before retrying broken + // // streams. + // Backoff func(int) time.Duration + // Logger provides enhanced logging capabilities. + Logger *grpclog.PrefixLogger +} + +// LoadReportingOptions contains configuration knobs for reporting load data. +type LoadReportingOptions struct { + LoadStore *load.Store +} + +// ErrResourceTypeUnsupported is an error used to indicate an unsupported xDS +// resource type. The wrapped ErrStr contains the details. +type ErrResourceTypeUnsupported struct { + ErrStr string +} + +// Error helps implements the error interface. +func (e ErrResourceTypeUnsupported) Error() string { + return e.ErrStr +} + +// VersionedClient is the interface to version specific operations of the +// client. +// +// It mainly deals with the type assertion from proto.Message to the real v2/v3 +// types, and grpc.Stream to the versioned stream types. +type VersionedClient interface { + // NewStream returns a new xDS client stream specific to the underlying + // transport protocol version. + NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) + // SendRequest constructs and sends out a DiscoveryRequest message specific + // to the underlying transport protocol version. + SendRequest(s grpc.ClientStream, resourceNames []string, rType resource.ResourceType, version, nonce, errMsg string) error + // RecvResponse uses the provided stream to receive a response specific to + // the underlying transport protocol version. + RecvResponse(s grpc.ClientStream) (proto.Message, error) + // ParseResponse type asserts message to the versioned response, and + // retrieves the fields. + ParseResponse(r proto.Message) (resource.ResourceType, []*anypb.Any, string, string, error) + + // The following are LRS methods. + + // NewLoadStatsStream returns a new LRS client stream specific to the + // underlying transport protocol version. + NewLoadStatsStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) + // SendFirstLoadStatsRequest constructs and sends the first request on the + // LRS stream. + SendFirstLoadStatsRequest(s grpc.ClientStream) error + // HandleLoadStatsResponse receives the first response from the server which + // contains the load reporting interval and the clusters for which the + // server asks the client to report load for. + // + // If the response sets SendAllClusters to true, the returned clusters is + // nil. + HandleLoadStatsResponse(s grpc.ClientStream) (clusters []string, _ time.Duration, _ error) + // SendLoadStatsRequest will be invoked at regular intervals to send load + // report with load data reported since the last time this method was + // invoked. + SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error +} diff --git a/xds/client/dump.go b/xds/client/dump.go new file mode 100644 index 0000000000..f01178b0ae --- /dev/null +++ b/xds/client/dump.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package client + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +func mergeMaps(maps []map[string]resource.UpdateWithMD) map[string]resource.UpdateWithMD { + ret := make(map[string]resource.UpdateWithMD) + for _, m := range maps { + for k, v := range m { + ret[k] = v + } + } + return ret +} + +func (c *clientImpl) dump(t resource.ResourceType) map[string]resource.UpdateWithMD { + c.authorityMu.Lock() + defer c.authorityMu.Unlock() + maps := make([]map[string]resource.UpdateWithMD, 0, len(c.authorities)) + for _, a := range c.authorities { + maps = append(maps, a.dump(t)) + } + return mergeMaps(maps) +} + +// DumpLDS returns the status and contents of LDS. +func (c *clientImpl) DumpLDS() map[string]resource.UpdateWithMD { + return c.dump(resource.ListenerResource) +} + +// DumpRDS returns the status and contents of RDS. +func (c *clientImpl) DumpRDS() map[string]resource.UpdateWithMD { + return c.dump(resource.RouteConfigResource) +} + +// DumpCDS returns the status and contents of CDS. +func (c *clientImpl) DumpCDS() map[string]resource.UpdateWithMD { + return c.dump(resource.ClusterResource) +} + +// DumpEDS returns the status and contents of EDS. +func (c *clientImpl) DumpEDS() map[string]resource.UpdateWithMD { + return c.dump(resource.EndpointsResource) +} diff --git a/xds/client/load/reporter.go b/xds/client/load/reporter.go new file mode 100644 index 0000000000..67e29e5bae --- /dev/null +++ b/xds/client/load/reporter.go @@ -0,0 +1,27 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package load + +// PerClusterReporter wraps the methods from the loadStore that are used here. +type PerClusterReporter interface { + CallStarted(locality string) + CallFinished(locality string, err error) + CallServerLoad(locality, name string, val float64) + CallDropped(category string) +} diff --git a/xds/client/load/store.go b/xds/client/load/store.go new file mode 100644 index 0000000000..551a5147b6 --- /dev/null +++ b/xds/client/load/store.go @@ -0,0 +1,426 @@ +/* + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package load provides functionality to record and maintain load data. +package load + +import ( + "sync" + "sync/atomic" + "time" +) + +const negativeOneUInt64 = ^uint64(0) + +// Store keeps the loads for multiple clusters and services to be reported via +// LRS. It contains loads to reported to one LRS server. Create multiple stores +// for multiple servers. +// +// It is safe for concurrent use. +type Store struct { + // mu only protects the map (2 layers). The read/write to *perClusterStore + // doesn't need to hold the mu. + mu sync.Mutex + // clusters is a map with cluster name as the key. The second layer is a map + // with service name as the key. Each value (perClusterStore) contains data + // for a (cluster, service) pair. + // + // Note that new entries are added to this map, but never removed. This is + // potentially a memory leak. But the memory is allocated for each new + // (cluster,service) pair, and the memory allocated is just pointers and + // maps. So this shouldn't get too bad. + clusters map[string]map[string]*perClusterStore +} + +// NewStore creates a Store. +func NewStore() *Store { + return &Store{ + clusters: make(map[string]map[string]*perClusterStore), + } +} + +// Stats returns the load data for the given cluster names. Data is returned in +// a slice with no specific order. +// +// If no clusterName is given (an empty slice), all data for all known clusters +// is returned. +// +// If a cluster's Data is empty (no load to report), it's not appended to the +// returned slice. +func (s *Store) Stats(clusterNames []string) []*Data { + var ret []*Data + s.mu.Lock() + defer s.mu.Unlock() + + if len(clusterNames) == 0 { + for _, c := range s.clusters { + ret = appendClusterStats(ret, c) + } + return ret + } + + for _, n := range clusterNames { + if c, ok := s.clusters[n]; ok { + ret = appendClusterStats(ret, c) + } + } + return ret +} + +// appendClusterStats gets Data for the given cluster, append to ret, and return +// the new slice. +// +// Data is only appended to ret if it's not empty. +func appendClusterStats(ret []*Data, cluster map[string]*perClusterStore) []*Data { + for _, d := range cluster { + data := d.stats() + if data == nil { + // Skip this data if it doesn't contain any information. + continue + } + ret = append(ret, data) + } + return ret +} + +// PerCluster returns the perClusterStore for the given clusterName + +// serviceName. +func (s *Store) PerCluster(clusterName, serviceName string) PerClusterReporter { + if s == nil { + return nil + } + + s.mu.Lock() + defer s.mu.Unlock() + c, ok := s.clusters[clusterName] + if !ok { + c = make(map[string]*perClusterStore) + s.clusters[clusterName] = c + } + + if p, ok := c[serviceName]; ok { + return p + } + p := &perClusterStore{ + cluster: clusterName, + service: serviceName, + } + c[serviceName] = p + return p +} + +// perClusterStore is a repository for LB policy implementations to report store +// load data. It contains load for a (cluster, edsService) pair. +// +// It is safe for concurrent use. +// +// TODO(easwars): Use regular maps with mutexes instead of sync.Map here. The +// latter is optimized for two common use cases: (1) when the entry for a given +// key is only ever written once but read many times, as in caches that only +// grow, or (2) when multiple goroutines read, write, and overwrite entries for +// disjoint sets of keys. In these two cases, use of a Map may significantly +// reduce lock contention compared to a Go map paired with a separate Mutex or +// RWMutex. +// Neither of these conditions are met here, and we should transition to a +// regular map with a mutex for better type safety. +type perClusterStore struct { + cluster, service string + drops sync.Map // map[string]*uint64 + localityRPCCount sync.Map // map[string]*rpcCountData + + mu sync.Mutex + lastLoadReportAt time.Time +} + +// Update functions are called by picker for each RPC. To avoid contention, all +// updates are done atomically. + +// CallDropped adds one drop record with the given category to store. +func (ls *perClusterStore) CallDropped(category string) { + if ls == nil { + return + } + + p, ok := ls.drops.Load(category) + if !ok { + tp := new(uint64) + p, _ = ls.drops.LoadOrStore(category, tp) + } + atomic.AddUint64(p.(*uint64), 1) +} + +// CallStarted adds one call started record for the given locality. +func (ls *perClusterStore) CallStarted(locality string) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + tp := newRPCCountData() + p, _ = ls.localityRPCCount.LoadOrStore(locality, tp) + } + p.(*rpcCountData).incrInProgress() +} + +// CallFinished adds one call finished record for the given locality. +// For successful calls, err needs to be nil. +func (ls *perClusterStore) CallFinished(locality string, err error) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for call-finish is not found should never happen. + return + } + p.(*rpcCountData).decrInProgress() + if err == nil { + p.(*rpcCountData).incrSucceeded() + } else { + p.(*rpcCountData).incrErrored() + } +} + +// CallServerLoad adds one server load record for the given locality. The +// load type is specified by desc, and its value by val. +func (ls *perClusterStore) CallServerLoad(locality, name string, d float64) { + if ls == nil { + return + } + + p, ok := ls.localityRPCCount.Load(locality) + if !ok { + // The map is never cleared, only values in the map are reset. So the + // case where entry for callServerLoad is not found should never happen. + return + } + p.(*rpcCountData).addServerLoad(name, d) +} + +// Data contains all load data reported to the Store since the most recent call +// to stats(). +type Data struct { + // Cluster is the name of the cluster this data is for. + Cluster string + // Service is the name of the EDS service this data is for. + Service string + // TotalDrops is the total number of dropped requests. + TotalDrops uint64 + // Drops is the number of dropped requests per category. + Drops map[string]uint64 + // LocalityStats contains load reports per locality. + LocalityStats map[string]LocalityData + // ReportInternal is the duration since last time load was reported (stats() + // was called). + ReportInterval time.Duration +} + +// LocalityData contains load data for a single locality. +type LocalityData struct { + // RequestStats contains counts of requests made to the locality. + RequestStats RequestData + // LoadStats contains server load data for requests made to the locality, + // indexed by the load type. + LoadStats map[string]ServerLoadData +} + +// RequestData contains request counts. +type RequestData struct { + // Succeeded is the number of succeeded requests. + Succeeded uint64 + // Errored is the number of requests which ran into errors. + Errored uint64 + // InProgress is the number of requests in flight. + InProgress uint64 +} + +// ServerLoadData contains server load data. +type ServerLoadData struct { + // Count is the number of load reports. + Count uint64 + // Sum is the total value of all load reports. + Sum float64 +} + +func newData(cluster, service string) *Data { + return &Data{ + Cluster: cluster, + Service: service, + Drops: make(map[string]uint64), + LocalityStats: make(map[string]LocalityData), + } +} + +// stats returns and resets all loads reported to the store, except inProgress +// rpc counts. +// +// It returns nil if the store doesn't contain any (new) data. +func (ls *perClusterStore) stats() *Data { + if ls == nil { + return nil + } + + sd := newData(ls.cluster, ls.service) + ls.drops.Range(func(key, val interface{}) bool { + d := atomic.SwapUint64(val.(*uint64), 0) + if d == 0 { + return true + } + sd.TotalDrops += d + keyStr := key.(string) + if keyStr != "" { + // Skip drops without category. They are counted in total_drops, but + // not in per category. One example is drops by circuit breaking. + sd.Drops[keyStr] = d + } + return true + }) + ls.localityRPCCount.Range(func(key, val interface{}) bool { + countData := val.(*rpcCountData) + succeeded := countData.loadAndClearSucceeded() + inProgress := countData.loadInProgress() + errored := countData.loadAndClearErrored() + if succeeded == 0 && inProgress == 0 && errored == 0 { + return true + } + + ld := LocalityData{ + RequestStats: RequestData{ + Succeeded: succeeded, + Errored: errored, + InProgress: inProgress, + }, + LoadStats: make(map[string]ServerLoadData), + } + countData.serverLoads.Range(func(key, val interface{}) bool { + sum, count := val.(*rpcLoadData).loadAndClear() + if count == 0 { + return true + } + ld.LoadStats[key.(string)] = ServerLoadData{ + Count: count, + Sum: sum, + } + return true + }) + sd.LocalityStats[key.(string)] = ld + return true + }) + + ls.mu.Lock() + sd.ReportInterval = time.Since(ls.lastLoadReportAt) + ls.lastLoadReportAt = time.Now() + ls.mu.Unlock() + + if sd.TotalDrops == 0 && len(sd.Drops) == 0 && len(sd.LocalityStats) == 0 { + return nil + } + return sd +} + +type rpcCountData struct { + // Only atomic accesses are allowed for the fields. + succeeded *uint64 + errored *uint64 + inProgress *uint64 + + // Map from load desc to load data (sum+count). Loading data from map is + // atomic, but updating data takes a lock, which could cause contention when + // multiple RPCs try to report loads for the same desc. + // + // To fix the contention, shard this map. + serverLoads sync.Map // map[string]*rpcLoadData +} + +func newRPCCountData() *rpcCountData { + return &rpcCountData{ + succeeded: new(uint64), + errored: new(uint64), + inProgress: new(uint64), + } +} + +func (rcd *rpcCountData) incrSucceeded() { + atomic.AddUint64(rcd.succeeded, 1) +} + +func (rcd *rpcCountData) loadAndClearSucceeded() uint64 { + return atomic.SwapUint64(rcd.succeeded, 0) +} + +func (rcd *rpcCountData) incrErrored() { + atomic.AddUint64(rcd.errored, 1) +} + +func (rcd *rpcCountData) loadAndClearErrored() uint64 { + return atomic.SwapUint64(rcd.errored, 0) +} + +func (rcd *rpcCountData) incrInProgress() { + atomic.AddUint64(rcd.inProgress, 1) +} + +func (rcd *rpcCountData) decrInProgress() { + atomic.AddUint64(rcd.inProgress, negativeOneUInt64) // atomic.Add(x, -1) +} + +func (rcd *rpcCountData) loadInProgress() uint64 { + return atomic.LoadUint64(rcd.inProgress) // InProgress count is not clear when reading. +} + +func (rcd *rpcCountData) addServerLoad(name string, d float64) { + loads, ok := rcd.serverLoads.Load(name) + if !ok { + tl := newRPCLoadData() + loads, _ = rcd.serverLoads.LoadOrStore(name, tl) + } + loads.(*rpcLoadData).add(d) +} + +// Data for server loads (from trailers or oob). Fields in this struct must be +// updated consistently. +// +// The current solution is to hold a lock, which could cause contention. To fix, +// shard serverLoads map in rpcCountData. +type rpcLoadData struct { + mu sync.Mutex + sum float64 + count uint64 +} + +func newRPCLoadData() *rpcLoadData { + return &rpcLoadData{} +} + +func (rld *rpcLoadData) add(v float64) { + rld.mu.Lock() + rld.sum += v + rld.count++ + rld.mu.Unlock() +} + +func (rld *rpcLoadData) loadAndClear() (s float64, c uint64) { + rld.mu.Lock() + s = rld.sum + rld.sum = 0 + c = rld.count + rld.count = 0 + rld.mu.Unlock() + return +} diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go new file mode 100644 index 0000000000..46568591f9 --- /dev/null +++ b/xds/client/load/store_test.go @@ -0,0 +1,446 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package load + +import ( + "fmt" + "sort" + "sync" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" +) + +var ( + dropCategories = []string{"drop_for_real", "drop_for_fun"} + localities = []string{"locality-A", "locality-B"} + errTest = fmt.Errorf("test error") +) + +// rpcData wraps the rpc counts and load data to be pushed to the store. +type rpcData struct { + start, success, failure int + serverData map[string]float64 // Will be reported with successful RPCs. +} + +// TestDrops spawns a bunch of goroutines which report drop data. After the +// goroutines have exited, the test dumps the stats from the Store and makes +// sure they are as expected. +func TestDrops(t *testing.T) { + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + "": 10, + } + wantStoreData = &Data{ + TotalDrops: 80, + Drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + } + ) + + ls := perClusterStore{} + var wg sync.WaitGroup + for category, count := range drops { + for i := 0; i < count; i++ { + wg.Add(1) + go func(c string) { + ls.CallDropped(c) + wg.Done() + }(category) + } + } + wg.Wait() + + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +// TestLocalityStats spawns a bunch of goroutines which report rpc and load +// data. After the goroutines have exited, the test dumps the stats from the +// Store and makes sure they are as expected. +func TestLocalityStats(t *testing.T) { + var ( + localityData = map[string]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &Data{ + LocalityStats: map[string]LocalityData{ + localities[0]: { + RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 20, Sum: 20}, + "disk": {Count: 20, Sum: 40}, + "cpu": {Count: 20, Sum: 60}, + "mem": {Count: 20, Sum: 80}, + }, + }, + localities[1]: { + RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 40, Sum: 40}, + "disk": {Count: 40, Sum: 80}, + "cpu": {Count: 40, Sum: 120}, + "mem": {Count: 40, Sum: 160}, + }, + }, + }, + } + ) + + ls := perClusterStore{} + var wg sync.WaitGroup + for locality, data := range localityData { + wg.Add(data.start) + for i := 0; i < data.start; i++ { + go func(l string) { + ls.CallStarted(l) + wg.Done() + }(locality) + } + // The calls to callStarted() need to happen before the other calls are + // made. Hence the wait here. + wg.Wait() + + wg.Add(data.success) + for i := 0; i < data.success; i++ { + go func(l string, serverData map[string]float64) { + ls.CallFinished(l, nil) + for n, d := range serverData { + ls.CallServerLoad(l, n, d) + } + wg.Done() + }(locality, data.serverData) + } + wg.Add(data.failure) + for i := 0; i < data.failure; i++ { + go func(l string) { + ls.CallFinished(l, errTest) + wg.Done() + }(locality) + } + wg.Wait() + } + + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +func TestResetAfterStats(t *testing.T) { + // Push a bunch of drops, call stats and load stats, and leave inProgress to be non-zero. + // Dump the stats. Verify expexted + // Push the same set of loads as before + // Now dump and verify the newly expected ones. + var ( + drops = map[string]int{ + dropCategories[0]: 30, + dropCategories[1]: 40, + } + localityData = map[string]rpcData{ + localities[0]: { + start: 40, + success: 20, + failure: 10, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + localities[1]: { + start: 80, + success: 40, + failure: 20, + serverData: map[string]float64{"net": 1, "disk": 2, "cpu": 3, "mem": 4}, + }, + } + wantStoreData = &Data{ + TotalDrops: 70, + Drops: map[string]uint64{ + dropCategories[0]: 30, + dropCategories[1]: 40, + }, + LocalityStats: map[string]LocalityData{ + localities[0]: { + RequestStats: RequestData{Succeeded: 20, Errored: 10, InProgress: 10}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 20, Sum: 20}, + "disk": {Count: 20, Sum: 40}, + "cpu": {Count: 20, Sum: 60}, + "mem": {Count: 20, Sum: 80}, + }, + }, + localities[1]: { + RequestStats: RequestData{Succeeded: 40, Errored: 20, InProgress: 20}, + LoadStats: map[string]ServerLoadData{ + "net": {Count: 40, Sum: 40}, + "disk": {Count: 40, Sum: 80}, + "cpu": {Count: 40, Sum: 120}, + "mem": {Count: 40, Sum: 160}, + }, + }, + }, + } + ) + + reportLoad := func(ls *perClusterStore) { + for category, count := range drops { + for i := 0; i < count; i++ { + ls.CallDropped(category) + } + } + for locality, data := range localityData { + for i := 0; i < data.start; i++ { + ls.CallStarted(locality) + } + for i := 0; i < data.success; i++ { + ls.CallFinished(locality, nil) + for n, d := range data.serverData { + ls.CallServerLoad(locality, n, d) + } + } + for i := 0; i < data.failure; i++ { + ls.CallFinished(locality, errTest) + } + } + } + + ls := perClusterStore{} + reportLoad(&ls) + gotStoreData := ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + // The above call to stats() should have reset all load reports except the + // inProgress rpc count. We are now going to push the same load data into + // the store. So, we should expect to see twice the count for inProgress. + for _, l := range localities { + ls := wantStoreData.LocalityStats[l] + ls.RequestStats.InProgress *= 2 + wantStoreData.LocalityStats[l] = ls + } + reportLoad(&ls) + gotStoreData = ls.stats() + if diff := cmp.Diff(wantStoreData, gotStoreData, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval")); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +var sortDataSlice = cmp.Transformer("SortDataSlice", func(in []*Data) []*Data { + out := append([]*Data(nil), in...) // Copy input to avoid mutating it + sort.Slice(out, + func(i, j int) bool { + if out[i].Cluster < out[j].Cluster { + return true + } + if out[i].Cluster == out[j].Cluster { + return out[i].Service < out[j].Service + } + return false + }, + ) + return out +}) + +// Test all load are returned for the given clusters, and all clusters are +// reported if no cluster is specified. +func TestStoreStats(t *testing.T) { + var ( + testClusters = []string{"c0", "c1", "c2"} + testServices = []string{"s0", "s1"} + testLocality = "test-locality" + ) + + store := NewStore() + for _, c := range testClusters { + for _, s := range testServices { + store.PerCluster(c, s).CallStarted(testLocality) + store.PerCluster(c, s).CallServerLoad(testLocality, "abc", 123) + store.PerCluster(c, s).CallDropped("dropped") + store.PerCluster(c, s).CallFinished(testLocality, nil) + } + } + + wantC0 := []*Data{ + { + Cluster: "c0", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c0", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + } + // Call Stats with just "c0", this should return data for "c0", and not + // touch data for other clusters. + gotC0 := store.Stats([]string{"c0"}) + if diff := cmp.Diff(wantC0, gotC0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + wantOther := []*Data{ + { + Cluster: "c1", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c1", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c2", Service: "s0", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + { + Cluster: "c2", Service: "s1", + TotalDrops: 1, Drops: map[string]uint64{"dropped": 1}, + LocalityStats: map[string]LocalityData{ + "test-locality": { + RequestStats: RequestData{Succeeded: 1}, + LoadStats: map[string]ServerLoadData{"abc": {Count: 1, Sum: 123}}, + }, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // remaining clusters, and not include c0 (because c0 data was cleared). + gotOther := store.Stats(nil) + if diff := cmp.Diff(wantOther, gotOther, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} + +// Test the cases that if a cluster doesn't have load to report, its data is not +// appended to the slice returned by Stats(). +func TestStoreStatsEmptyDataNotReported(t *testing.T) { + var ( + testServices = []string{"s0", "s1"} + testLocality = "test-locality" + ) + + store := NewStore() + // "c0"'s RPCs all finish with success. + for _, s := range testServices { + store.PerCluster("c0", s).CallStarted(testLocality) + store.PerCluster("c0", s).CallFinished(testLocality, nil) + } + // "c1"'s RPCs never finish (always inprocess). + for _, s := range testServices { + store.PerCluster("c1", s).CallStarted(testLocality) + } + + want0 := []*Data{ + { + Cluster: "c0", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{Succeeded: 1}}, + }, + }, + { + Cluster: "c0", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{Succeeded: 1}}, + }, + }, + { + Cluster: "c1", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + { + Cluster: "c1", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + } + // Call Stats with empty slice, this should return data for all the + // clusters. + got0 := store.Stats(nil) + if diff := cmp.Diff(want0, got0, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } + + want1 := []*Data{ + { + Cluster: "c1", Service: "s0", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + { + Cluster: "c1", Service: "s1", + LocalityStats: map[string]LocalityData{ + "test-locality": {RequestStats: RequestData{InProgress: 1}}, + }, + }, + } + // Call Stats with empty slice again, this should return data only for "c1", + // because "c0" data was cleared, but "c1" has in-progress RPCs. + got1 := store.Stats(nil) + if diff := cmp.Diff(want1, got1, cmpopts.EquateEmpty(), cmpopts.IgnoreFields(Data{}, "ReportInterval"), sortDataSlice); diff != "" { + t.Errorf("store.stats() returned unexpected diff (-want +got):\n%s", diff) + } +} diff --git a/xds/client/loadreport.go b/xds/client/loadreport.go new file mode 100644 index 0000000000..bb45e013dc --- /dev/null +++ b/xds/client/loadreport.go @@ -0,0 +1,47 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// ReportLoad starts an load reporting stream to the given server. If the server +// is not an empty string, and is different from the management server, a new +// ClientConn will be created. +// +// The same options used for creating the Client will be used (including +// NodeProto, and dial options if necessary). +// +// It returns a Store for the user to report loads, a function to cancel the +// load reporting stream. +func (c *clientImpl) ReportLoad(server string) (*load.Store, func()) { + // TODO: load reporting with federation also needs find the authority for + // this server first, then reports load to it. Currently always report to + // the default authority. This is needed to avoid a nil pointer panic. + a, unref, err := c.findAuthority(resource.ParseName("")) + if err != nil { + return nil, func() {} + } + store, cancelF := a.reportLoad(server) + return store, func() { + cancelF() + unref() + } +} diff --git a/xds/client/logging.go b/xds/client/logging.go new file mode 100644 index 0000000000..2b803f3c70 --- /dev/null +++ b/xds/client/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package client + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[xds-client %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *clientImpl) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go new file mode 100644 index 0000000000..7a8b82f425 --- /dev/null +++ b/xds/client/pubsub/dump.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pubsub + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + anypb "github.com/golang/protobuf/ptypes/any" +) + +func rawFromCache(s string, cache interface{}) *anypb.Any { + switch c := cache.(type) { + case map[string]resource.ListenerUpdate: + if v, ok := c[s]; ok { + return v.Raw + } + return nil + case map[string]resource.RouteConfigUpdate: + if v, ok := c[s]; ok { + return v.Raw + } + return nil + case map[string]resource.ClusterUpdate: + if v, ok := c[s]; ok { + return v.Raw + } + return nil + case map[string]resource.EndpointsUpdate: + if v, ok := c[s]; ok { + return v.Raw + } + return nil + default: + return nil + } +} + +// Dump dumps the resource for the given type. +func (pb *Pubsub) Dump(t resource.ResourceType) map[string]resource.UpdateWithMD { + pb.mu.Lock() + defer pb.mu.Unlock() + + var ( + md map[string]resource.UpdateMetadata + cache interface{} + ) + switch t { + case resource.ListenerResource: + md = pb.ldsMD + cache = pb.ldsCache + case resource.RouteConfigResource: + md = pb.rdsMD + cache = pb.rdsCache + case resource.ClusterResource: + md = pb.cdsMD + cache = pb.cdsCache + case resource.EndpointsResource: + md = pb.edsMD + cache = pb.edsCache + default: + pb.logger.Errorf("dumping resource of unknown type: %v", t) + return nil + } + + ret := make(map[string]resource.UpdateWithMD, len(md)) + for s, md := range md { + ret[s] = resource.UpdateWithMD{ + MD: md, + Raw: rawFromCache(s, cache), + } + } + return ret +} diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go new file mode 100644 index 0000000000..4226f581cf --- /dev/null +++ b/xds/client/pubsub/interface.go @@ -0,0 +1,39 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pubsub + +import "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + +// UpdateHandler receives and processes (by taking appropriate actions) xDS +// resource updates from an APIClient for a specific version. +// +// It's a subset of the APIs of a *Pubsub. +type UpdateHandler interface { + // NewListeners handles updates to xDS listener resources. + NewListeners(map[string]resource.ListenerUpdateErrTuple, resource.UpdateMetadata) + // NewRouteConfigs handles updates to xDS RouteConfiguration resources. + NewRouteConfigs(map[string]resource.RouteConfigUpdateErrTuple, resource.UpdateMetadata) + // NewClusters handles updates to xDS Cluster resources. + NewClusters(map[string]resource.ClusterUpdateErrTuple, resource.UpdateMetadata) + // NewEndpoints handles updates to xDS ClusterLoadAssignment (or tersely + // referred to as Endpoints) resources. + NewEndpoints(map[string]resource.EndpointsUpdateErrTuple, resource.UpdateMetadata) + // NewConnectionError handles connection errors from the xDS stream. The + // error will be reported to all the resource watchers. + NewConnectionError(err error) +} diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go new file mode 100644 index 0000000000..d4412fd4ac --- /dev/null +++ b/xds/client/pubsub/pubsub.go @@ -0,0 +1,182 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package pubsub implements a utility type to maintain resource watchers and +// the updates. +// +// This package is designed to work with the xds resources. It could be made a +// general system that works with all types. +package pubsub + +import ( + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" +) + +// Pubsub maintains resource watchers and resource updates. +// +// There can be multiple watchers for the same resource. An update to a resource +// triggers updates to all the existing watchers. Watchers can be canceled at +// any time. +type Pubsub struct { + done *grpcsync.Event + logger *grpclog.PrefixLogger + watchExpiryTimeout time.Duration + + updateCh *buffer.Unbounded // chan *watcherInfoWithUpdate + // All the following maps are to keep the updates/metadata in a cache. + mu sync.Mutex + ldsWatchers map[string]map[*watchInfo]bool + ldsCache map[string]resource.ListenerUpdate + ldsMD map[string]resource.UpdateMetadata + rdsWatchers map[string]map[*watchInfo]bool + rdsCache map[string]resource.RouteConfigUpdate + rdsMD map[string]resource.UpdateMetadata + cdsWatchers map[string]map[*watchInfo]bool + cdsCache map[string]resource.ClusterUpdate + cdsMD map[string]resource.UpdateMetadata + edsWatchers map[string]map[*watchInfo]bool + edsCache map[string]resource.EndpointsUpdate + edsMD map[string]resource.UpdateMetadata +} + +// New creates a new Pubsub. +func New(watchExpiryTimeout time.Duration, logger *grpclog.PrefixLogger) *Pubsub { + pb := &Pubsub{ + done: grpcsync.NewEvent(), + logger: logger, + watchExpiryTimeout: watchExpiryTimeout, + + updateCh: buffer.NewUnbounded(), + ldsWatchers: make(map[string]map[*watchInfo]bool), + ldsCache: make(map[string]resource.ListenerUpdate), + ldsMD: make(map[string]resource.UpdateMetadata), + rdsWatchers: make(map[string]map[*watchInfo]bool), + rdsCache: make(map[string]resource.RouteConfigUpdate), + rdsMD: make(map[string]resource.UpdateMetadata), + cdsWatchers: make(map[string]map[*watchInfo]bool), + cdsCache: make(map[string]resource.ClusterUpdate), + cdsMD: make(map[string]resource.UpdateMetadata), + edsWatchers: make(map[string]map[*watchInfo]bool), + edsCache: make(map[string]resource.EndpointsUpdate), + edsMD: make(map[string]resource.UpdateMetadata), + } + go pb.run() + return pb +} + +// WatchListener registers a watcher for the LDS resource. +// +// It also returns whether this is the first watch for this resource. +func (pb *Pubsub) WatchListener(serviceName string, cb func(resource.ListenerUpdate, error)) (first bool, cancel func() bool) { + wi := &watchInfo{ + c: pb, + rType: resource.ListenerResource, + target: serviceName, + ldsCallback: cb, + } + + wi.expiryTimer = time.AfterFunc(pb.watchExpiryTimeout, func() { + wi.timeout() + }) + return pb.watch(wi) +} + +// WatchRouteConfig register a watcher for the RDS resource. +// +// It also returns whether this is the first watch for this resource. +func (pb *Pubsub) WatchRouteConfig(routeName string, cb func(resource.RouteConfigUpdate, error)) (first bool, cancel func() bool) { + wi := &watchInfo{ + c: pb, + rType: resource.RouteConfigResource, + target: routeName, + rdsCallback: cb, + } + + wi.expiryTimer = time.AfterFunc(pb.watchExpiryTimeout, func() { + wi.timeout() + }) + return pb.watch(wi) +} + +// WatchCluster register a watcher for the CDS resource. +// +// It also returns whether this is the first watch for this resource. +func (pb *Pubsub) WatchCluster(clusterName string, cb func(resource.ClusterUpdate, error)) (first bool, cancel func() bool) { + wi := &watchInfo{ + c: pb, + rType: resource.ClusterResource, + target: clusterName, + cdsCallback: cb, + } + + wi.expiryTimer = time.AfterFunc(pb.watchExpiryTimeout, func() { + wi.timeout() + }) + return pb.watch(wi) +} + +// WatchEndpoints registers a watcher for the EDS resource. +// +// It also returns whether this is the first watch for this resource. +func (pb *Pubsub) WatchEndpoints(clusterName string, cb func(resource.EndpointsUpdate, error)) (first bool, cancel func() bool) { + wi := &watchInfo{ + c: pb, + rType: resource.EndpointsResource, + target: clusterName, + edsCallback: cb, + } + + wi.expiryTimer = time.AfterFunc(pb.watchExpiryTimeout, func() { + wi.timeout() + }) + return pb.watch(wi) +} + +// Close closes the pubsub. +func (pb *Pubsub) Close() { + if pb.done.HasFired() { + return + } + pb.done.Fire() +} + +// run is a goroutine for all the callbacks. +// +// Callback can be called in watch(), if an item is found in cache. Without this +// goroutine, the callback will be called inline, which might cause a deadlock +// in user's code. Callbacks also cannot be simple `go callback()` because the +// order matters. +func (pb *Pubsub) run() { + for { + select { + case t := <-pb.updateCh.Get(): + pb.updateCh.Load() + if pb.done.HasFired() { + return + } + pb.callCallback(t.(*watcherInfoWithUpdate)) + case <-pb.done.Done(): + return + } + } +} diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go new file mode 100644 index 0000000000..42f8a7bd57 --- /dev/null +++ b/xds/client/pubsub/update.go @@ -0,0 +1,318 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pubsub + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + "google.golang.org/protobuf/proto" +) + +type watcherInfoWithUpdate struct { + wi *watchInfo + update interface{} + err error +} + +// scheduleCallback should only be called by methods of watchInfo, which checks +// for watcher states and maintain consistency. +func (pb *Pubsub) scheduleCallback(wi *watchInfo, update interface{}, err error) { + pb.updateCh.Put(&watcherInfoWithUpdate{ + wi: wi, + update: update, + err: err, + }) +} + +func (pb *Pubsub) callCallback(wiu *watcherInfoWithUpdate) { + pb.mu.Lock() + // Use a closure to capture the callback and type assertion, to save one + // more switch case. + // + // The callback must be called without pb.mu. Otherwise if the callback calls + // another watch() inline, it will cause a deadlock. This leaves a small + // window that a watcher's callback could be called after the watcher is + // canceled, and the user needs to take care of it. + var ccb func() + switch wiu.wi.rType { + case resource.ListenerResource: + if s, ok := pb.ldsWatchers[wiu.wi.target]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.ldsCallback(wiu.update.(resource.ListenerUpdate), wiu.err) } + } + case resource.RouteConfigResource: + if s, ok := pb.rdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.rdsCallback(wiu.update.(resource.RouteConfigUpdate), wiu.err) } + } + case resource.ClusterResource: + if s, ok := pb.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.cdsCallback(wiu.update.(resource.ClusterUpdate), wiu.err) } + } + case resource.EndpointsResource: + if s, ok := pb.edsWatchers[wiu.wi.target]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.edsCallback(wiu.update.(resource.EndpointsUpdate), wiu.err) } + } + } + pb.mu.Unlock() + + if ccb != nil { + ccb() + } +} + +// NewListeners is called when there's a new LDS update. +func (pb *Pubsub) NewListeners(updates map[string]resource.ListenerUpdateErrTuple, metadata resource.UpdateMetadata) { + pb.mu.Lock() + defer pb.mu.Unlock() + + for name, uErr := range updates { + if s, ok := pb.ldsWatchers[name]; ok { + if uErr.Err != nil { + // On error, keep previous version for each resource. But update + // status and error. + mdCopy := pb.ldsMD[name] + mdCopy.ErrState = metadata.ErrState + mdCopy.Status = metadata.Status + pb.ldsMD[name] = mdCopy + for wi := range s { + wi.newError(uErr.Err) + } + continue + } + // If we get here, it means that the update is a valid one. Notify + // watchers only if this is a first time update or it is different + // from the one currently cached. + if cur, ok := pb.ldsCache[name]; !ok || !proto.Equal(cur.Raw, uErr.Update.Raw) { + for wi := range s { + wi.newUpdate(uErr.Update) + } + } + // Sync cache. + pb.logger.Debugf("LDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(uErr)) + pb.ldsCache[name] = uErr.Update + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + mdCopy := metadata + mdCopy.Status = resource.ServiceStatusACKed + mdCopy.ErrState = nil + if metadata.ErrState != nil { + mdCopy.Version = metadata.ErrState.Version + } + pb.ldsMD[name] = mdCopy + } + } + // Resources not in the new update were removed by the server, so delete + // them. + for name := range pb.ldsCache { + if _, ok := updates[name]; !ok { + // If resource exists in cache, but not in the new update, delete + // the resource from cache, and also send an resource not found + // error to indicate resource removed. + delete(pb.ldsCache, name) + pb.ldsMD[name] = resource.UpdateMetadata{Status: resource.ServiceStatusNotExist} + for wi := range pb.ldsWatchers[name] { + wi.resourceNotFound() + } + } + } + // When LDS resource is removed, we don't delete corresponding RDS cached + // data. The RDS watch will be canceled, and cache entry is removed when the + // last watch is canceled. +} + +// NewRouteConfigs is called when there's a new RDS update. +func (pb *Pubsub) NewRouteConfigs(updates map[string]resource.RouteConfigUpdateErrTuple, metadata resource.UpdateMetadata) { + pb.mu.Lock() + defer pb.mu.Unlock() + + // If no error received, the status is ACK. + for name, uErr := range updates { + if s, ok := pb.rdsWatchers[name]; ok { + if uErr.Err != nil { + // On error, keep previous version for each resource. But update + // status and error. + mdCopy := pb.rdsMD[name] + mdCopy.ErrState = metadata.ErrState + mdCopy.Status = metadata.Status + pb.rdsMD[name] = mdCopy + for wi := range s { + wi.newError(uErr.Err) + } + continue + } + // If we get here, it means that the update is a valid one. Notify + // watchers only if this is a first time update or it is different + // from the one currently cached. + if cur, ok := pb.rdsCache[name]; !ok || !proto.Equal(cur.Raw, uErr.Update.Raw) { + for wi := range s { + wi.newUpdate(uErr.Update) + } + } + // Sync cache. + pb.logger.Debugf("RDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(uErr)) + pb.rdsCache[name] = uErr.Update + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + mdCopy := metadata + mdCopy.Status = resource.ServiceStatusACKed + mdCopy.ErrState = nil + if metadata.ErrState != nil { + mdCopy.Version = metadata.ErrState.Version + } + pb.rdsMD[name] = mdCopy + } + } +} + +// NewClusters is called when there's a new CDS update. +func (pb *Pubsub) NewClusters(updates map[string]resource.ClusterUpdateErrTuple, metadata resource.UpdateMetadata) { + pb.mu.Lock() + defer pb.mu.Unlock() + + for name, uErr := range updates { + if s, ok := pb.cdsWatchers[name]; ok { + if uErr.Err != nil { + // On error, keep previous version for each resource. But update + // status and error. + mdCopy := pb.cdsMD[name] + mdCopy.ErrState = metadata.ErrState + mdCopy.Status = metadata.Status + pb.cdsMD[name] = mdCopy + for wi := range s { + // Send the watcher the individual error, instead of the + // overall combined error from the metadata.ErrState. + wi.newError(uErr.Err) + } + continue + } + // If we get here, it means that the update is a valid one. Notify + // watchers only if this is a first time update or it is different + // from the one currently cached. + if cur, ok := pb.cdsCache[name]; !ok || !proto.Equal(cur.Raw, uErr.Update.Raw) { + for wi := range s { + wi.newUpdate(uErr.Update) + } + } + // Sync cache. + pb.logger.Debugf("CDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(uErr)) + pb.cdsCache[name] = uErr.Update + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + mdCopy := metadata + mdCopy.Status = resource.ServiceStatusACKed + mdCopy.ErrState = nil + if metadata.ErrState != nil { + mdCopy.Version = metadata.ErrState.Version + } + pb.cdsMD[name] = mdCopy + } + } + // Resources not in the new update were removed by the server, so delete + // them. + for name := range pb.cdsCache { + if _, ok := updates[name]; !ok { + // If resource exists in cache, but not in the new update, delete it + // from cache, and also send an resource not found error to indicate + // resource removed. + delete(pb.cdsCache, name) + pb.ldsMD[name] = resource.UpdateMetadata{Status: resource.ServiceStatusNotExist} + for wi := range pb.cdsWatchers[name] { + wi.resourceNotFound() + } + } + } + // When CDS resource is removed, we don't delete corresponding EDS cached + // data. The EDS watch will be canceled, and cache entry is removed when the + // last watch is canceled. +} + +// NewEndpoints is called when there's anew EDS update. +func (pb *Pubsub) NewEndpoints(updates map[string]resource.EndpointsUpdateErrTuple, metadata resource.UpdateMetadata) { + pb.mu.Lock() + defer pb.mu.Unlock() + + for name, uErr := range updates { + if s, ok := pb.edsWatchers[name]; ok { + if uErr.Err != nil { + // On error, keep previous version for each resource. But update + // status and error. + mdCopy := pb.edsMD[name] + mdCopy.ErrState = metadata.ErrState + mdCopy.Status = metadata.Status + pb.edsMD[name] = mdCopy + for wi := range s { + // Send the watcher the individual error, instead of the + // overall combined error from the metadata.ErrState. + wi.newError(uErr.Err) + } + continue + } + // If we get here, it means that the update is a valid one. Notify + // watchers only if this is a first time update or it is different + // from the one currently cached. + if cur, ok := pb.edsCache[name]; !ok || !proto.Equal(cur.Raw, uErr.Update.Raw) { + for wi := range s { + wi.newUpdate(uErr.Update) + } + } + // Sync cache. + pb.logger.Debugf("EDS resource with name %v, value %+v added to cache", name, pretty.ToJSON(uErr)) + pb.edsCache[name] = uErr.Update + // Set status to ACK, and clear error state. The metadata might be a + // NACK metadata because some other resources in the same response + // are invalid. + mdCopy := metadata + mdCopy.Status = resource.ServiceStatusACKed + mdCopy.ErrState = nil + if metadata.ErrState != nil { + mdCopy.Version = metadata.ErrState.Version + } + pb.edsMD[name] = mdCopy + } + } +} + +// NewConnectionError is called by the underlying xdsAPIClient when it receives +// a connection error. The error will be forwarded to all the resource watchers. +func (pb *Pubsub) NewConnectionError(err error) { + pb.mu.Lock() + defer pb.mu.Unlock() + + for _, s := range pb.ldsWatchers { + for wi := range s { + wi.newError(resource.NewErrorf(resource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err)) + } + } + for _, s := range pb.rdsWatchers { + for wi := range s { + wi.newError(resource.NewErrorf(resource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err)) + } + } + for _, s := range pb.cdsWatchers { + for wi := range s { + wi.newError(resource.NewErrorf(resource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err)) + } + } + for _, s := range pb.edsWatchers { + for wi := range s { + wi.newError(resource.NewErrorf(resource.ErrorTypeConnection, "xds: error received from xDS stream: %v", err)) + } + } +} diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go new file mode 100644 index 0000000000..13cd8ed885 --- /dev/null +++ b/xds/client/pubsub/watch.go @@ -0,0 +1,232 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package pubsub + +import ( + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + +type watchInfoState int + +const ( + watchInfoStateStarted watchInfoState = iota + watchInfoStateRespReceived + watchInfoStateTimeout + watchInfoStateCanceled +) + +// watchInfo holds all the information from a watch() call. +type watchInfo struct { + c *Pubsub + rType resource.ResourceType + target string + + ldsCallback func(resource.ListenerUpdate, error) + rdsCallback func(resource.RouteConfigUpdate, error) + cdsCallback func(resource.ClusterUpdate, error) + edsCallback func(resource.EndpointsUpdate, error) + + expiryTimer *time.Timer + + // mu protects state, and c.scheduleCallback(). + // - No callback should be scheduled after watchInfo is canceled. + // - No timeout error should be scheduled after watchInfo is resp received. + mu sync.Mutex + state watchInfoState +} + +func (wi *watchInfo) newUpdate(update interface{}) { + wi.mu.Lock() + defer wi.mu.Unlock() + if wi.state == watchInfoStateCanceled { + return + } + wi.state = watchInfoStateRespReceived + wi.expiryTimer.Stop() + wi.c.scheduleCallback(wi, update, nil) +} + +func (wi *watchInfo) newError(err error) { + wi.mu.Lock() + defer wi.mu.Unlock() + if wi.state == watchInfoStateCanceled { + return + } + wi.state = watchInfoStateRespReceived + wi.expiryTimer.Stop() + wi.sendErrorLocked(err) +} + +func (wi *watchInfo) resourceNotFound() { + wi.mu.Lock() + defer wi.mu.Unlock() + if wi.state == watchInfoStateCanceled { + return + } + wi.state = watchInfoStateRespReceived + wi.expiryTimer.Stop() + wi.sendErrorLocked(resource.NewErrorf(resource.ErrorTypeResourceNotFound, "xds: %v target %s not found in received response", wi.rType, wi.target)) +} + +func (wi *watchInfo) timeout() { + wi.mu.Lock() + defer wi.mu.Unlock() + if wi.state == watchInfoStateCanceled || wi.state == watchInfoStateRespReceived { + return + } + wi.state = watchInfoStateTimeout + wi.sendErrorLocked(fmt.Errorf("xds: %v target %s not found, watcher timeout", wi.rType, wi.target)) +} + +// Caller must hold wi.mu. +func (wi *watchInfo) sendErrorLocked(err error) { + var ( + u interface{} + ) + switch wi.rType { + case resource.ListenerResource: + u = resource.ListenerUpdate{} + case resource.RouteConfigResource: + u = resource.RouteConfigUpdate{} + case resource.ClusterResource: + u = resource.ClusterUpdate{} + case resource.EndpointsResource: + u = resource.EndpointsUpdate{} + } + wi.c.scheduleCallback(wi, u, err) +} + +func (wi *watchInfo) cancel() { + wi.mu.Lock() + defer wi.mu.Unlock() + if wi.state == watchInfoStateCanceled { + return + } + wi.expiryTimer.Stop() + wi.state = watchInfoStateCanceled +} + +func (pb *Pubsub) watch(wi *watchInfo) (first bool, cancel func() bool) { + pb.mu.Lock() + defer pb.mu.Unlock() + pb.logger.Debugf("new watch for type %v, resource name %v", wi.rType, wi.target) + var ( + watchers map[string]map[*watchInfo]bool + mds map[string]resource.UpdateMetadata + ) + switch wi.rType { + case resource.ListenerResource: + watchers = pb.ldsWatchers + mds = pb.ldsMD + case resource.RouteConfigResource: + watchers = pb.rdsWatchers + mds = pb.rdsMD + case resource.ClusterResource: + watchers = pb.cdsWatchers + mds = pb.cdsMD + case resource.EndpointsResource: + watchers = pb.edsWatchers + mds = pb.edsMD + default: + pb.logger.Errorf("unknown watch type: %v", wi.rType) + return false, nil + } + + var firstWatcher bool + resourceName := wi.target + s, ok := watchers[wi.target] + if !ok { + // If this is a new watcher, will ask lower level to send a new request + // with the resource name. + // + // If this (type+name) is already being watched, will not notify the + // underlying versioned apiClient. + pb.logger.Debugf("first watch for type %v, resource name %v, will send a new xDS request", wi.rType, wi.target) + s = make(map[*watchInfo]bool) + watchers[resourceName] = s + mds[resourceName] = resource.UpdateMetadata{Status: resource.ServiceStatusRequested} + firstWatcher = true + } + // No matter what, add the new watcher to the set, so it's callback will be + // call for new responses. + s[wi] = true + + // If the resource is in cache, call the callback with the value. + switch wi.rType { + case resource.ListenerResource: + if v, ok := pb.ldsCache[resourceName]; ok { + pb.logger.Debugf("LDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) + wi.newUpdate(v) + } + case resource.RouteConfigResource: + if v, ok := pb.rdsCache[resourceName]; ok { + pb.logger.Debugf("RDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) + wi.newUpdate(v) + } + case resource.ClusterResource: + if v, ok := pb.cdsCache[resourceName]; ok { + pb.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) + wi.newUpdate(v) + } + case resource.EndpointsResource: + if v, ok := pb.edsCache[resourceName]; ok { + pb.logger.Debugf("EDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) + wi.newUpdate(v) + } + } + + return firstWatcher, func() bool { + pb.logger.Debugf("watch for type %v, resource name %v canceled", wi.rType, wi.target) + wi.cancel() + pb.mu.Lock() + defer pb.mu.Unlock() + var lastWatcher bool + if s := watchers[resourceName]; s != nil { + // Remove this watcher, so it's callback will not be called in the + // future. + delete(s, wi) + if len(s) == 0 { + pb.logger.Debugf("last watch for type %v, resource name %v canceled, will send a new xDS request", wi.rType, wi.target) + // If this was the last watcher, also tell xdsv2Client to stop + // watching this resource. + delete(watchers, resourceName) + delete(mds, resourceName) + lastWatcher = true + // Remove the resource from cache. When a watch for this + // resource is added later, it will trigger a xDS request with + // resource names, and client will receive new xDS responses. + switch wi.rType { + case resource.ListenerResource: + delete(pb.ldsCache, resourceName) + case resource.RouteConfigResource: + delete(pb.rdsCache, resourceName) + case resource.ClusterResource: + delete(pb.cdsCache, resourceName) + case resource.EndpointsResource: + delete(pb.edsCache, resourceName) + } + } + } + return lastWatcher + } +} diff --git a/xds/client/requests_counter.go b/xds/client/requests_counter.go new file mode 100644 index 0000000000..04ee728dc8 --- /dev/null +++ b/xds/client/requests_counter.go @@ -0,0 +1,107 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package client + +import ( + "fmt" + "sync" + "sync/atomic" +) + +type clusterNameAndServiceName struct { + clusterName, edsServcieName string +} + +type clusterRequestsCounter struct { + mu sync.Mutex + clusters map[clusterNameAndServiceName]*ClusterRequestsCounter +} + +var src = &clusterRequestsCounter{ + clusters: make(map[clusterNameAndServiceName]*ClusterRequestsCounter), +} + +// ClusterRequestsCounter is used to track the total inflight requests for a +// service with the provided name. +type ClusterRequestsCounter struct { + ClusterName string + EDSServiceName string + numRequests uint32 +} + +// GetClusterRequestsCounter returns the ClusterRequestsCounter with the +// provided serviceName. If one does not exist, it creates it. +func GetClusterRequestsCounter(clusterName, edsServiceName string) *ClusterRequestsCounter { + src.mu.Lock() + defer src.mu.Unlock() + k := clusterNameAndServiceName{ + clusterName: clusterName, + edsServcieName: edsServiceName, + } + c, ok := src.clusters[k] + if !ok { + c = &ClusterRequestsCounter{ClusterName: clusterName} + src.clusters[k] = c + } + return c +} + +// StartRequest starts a request for a cluster, incrementing its number of +// requests by 1. Returns an error if the max number of requests is exceeded. +func (c *ClusterRequestsCounter) StartRequest(max uint32) error { + // Note that during race, the limits could be exceeded. This is allowed: + // "Since the implementation is eventually consistent, races between threads + // may allow limits to be potentially exceeded." + // https://www.envoyproxy.io/docs/envoy/latest/intro/arch_overview/upstream/circuit_breaking#arch-overview-circuit-break. + if atomic.LoadUint32(&c.numRequests) >= max { + return fmt.Errorf("max requests %v exceeded on service %v", max, c.ClusterName) + } + atomic.AddUint32(&c.numRequests, 1) + return nil +} + +// EndRequest ends a request for a service, decrementing its number of requests +// by 1. +func (c *ClusterRequestsCounter) EndRequest() { + atomic.AddUint32(&c.numRequests, ^uint32(0)) +} + +// ClearCounterForTesting clears the counter for the service. Should be only +// used in tests. +func ClearCounterForTesting(clusterName, edsServiceName string) { + src.mu.Lock() + defer src.mu.Unlock() + k := clusterNameAndServiceName{ + clusterName: clusterName, + edsServcieName: edsServiceName, + } + c, ok := src.clusters[k] + if !ok { + return + } + c.numRequests = 0 +} + +// ClearAllCountersForTesting clears all the counters. Should be only used in +// tests. +func ClearAllCountersForTesting() { + src.mu.Lock() + defer src.mu.Unlock() + src.clusters = make(map[clusterNameAndServiceName]*ClusterRequestsCounter) +} diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go new file mode 100644 index 0000000000..be3269290e --- /dev/null +++ b/xds/client/resource/errors.go @@ -0,0 +1,60 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resource + +import "fmt" + +// ErrorType is the type of the error that the watcher will receive from the xds +// client. +type ErrorType int + +const ( + // ErrorTypeUnknown indicates the error doesn't have a specific type. It is + // the default value, and is returned if the error is not an xds error. + ErrorTypeUnknown ErrorType = iota + // ErrorTypeConnection indicates a connection error from the gRPC client. + ErrorTypeConnection + // ErrorTypeResourceNotFound indicates a resource is not found from the xds + // response. It's typically returned if the resource is removed in the xds + // server. + ErrorTypeResourceNotFound +) + +type xdsClientError struct { + t ErrorType + desc string +} + +func (e *xdsClientError) Error() string { + return e.desc +} + +// NewErrorf creates an xds client error. The callbacks are called with this +// error, to pass additional information about the error. +func NewErrorf(t ErrorType, format string, args ...interface{}) error { + return &xdsClientError{t: t, desc: fmt.Sprintf(format, args...)} +} + +// ErrType returns the error's type. +func ErrType(e error) ErrorType { + if xe, ok := e.(*xdsClientError); ok { + return xe.t + } + return ErrorTypeUnknown +} diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go new file mode 100644 index 0000000000..ccc786233e --- /dev/null +++ b/xds/client/resource/filter_chain.go @@ -0,0 +1,872 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "errors" + "fmt" + "net" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" +) + +const ( + // Used as the map key for unspecified prefixes. The actual value of this + // key is immaterial. + unspecifiedPrefixMapKey = "unspecified" + + // An unspecified destination or source prefix should be considered a less + // specific match than a wildcard prefix, `0.0.0.0/0` or `::/0`. Also, an + // unspecified prefix should match most v4 and v6 addresses compared to the + // wildcard prefixes which match only a specific network (v4 or v6). + // + // We use these constants when looking up the most specific prefix match. A + // wildcard prefix will match 0 bits, and to make sure that a wildcard + // prefix is considered a more specific match than an unspecified prefix, we + // use a value of -1 for the latter. + noPrefixMatch = -2 + unspecifiedPrefixMatch = -1 +) + +// FilterChain captures information from within a FilterChain message in a +// Listener resource. +type FilterChain struct { + // SecurityCfg contains transport socket security configuration. + SecurityCfg *SecurityConfig + // HTTPFilters represent the HTTP Filters that comprise this FilterChain. + HTTPFilters []HTTPFilter + // RouteConfigName is the route configuration name for this FilterChain. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + RouteConfigName string + // InlineRouteConfig is the inline route configuration (RDS response) + // returned for this filter chain. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + InlineRouteConfig *RouteConfigUpdate +} + +// VirtualHostWithInterceptors captures information present in a VirtualHost +// update, and also contains routes with instantiated HTTP Filters. +type VirtualHostWithInterceptors struct { + // Domains are the domain names which map to this Virtual Host. On the + // server side, this will be dictated by the :authority header of the + // incoming RPC. + Domains []string + // Routes are the Routes for this Virtual Host. + Routes []RouteWithInterceptors +} + +// RouteWithInterceptors captures information in a Route, and contains +// a usable matcher and also instantiated HTTP Filters. +type RouteWithInterceptors struct { + // M is the matcher used to match to this route. + M *CompositeMatcher + // ActionType is the type of routing action to initiate once matched to. + ActionType RouteActionType + // Interceptors are interceptors instantiated for this route. These will be + // constructed from a combination of the top level configuration and any + // HTTP Filter overrides present in Virtual Host or Route. + Interceptors []resolver.ServerInterceptor +} + +// ConstructUsableRouteConfiguration takes Route Configuration and converts it +// into matchable route configuration, with instantiated HTTP Filters per route. +func (f *FilterChain) ConstructUsableRouteConfiguration(config RouteConfigUpdate) ([]VirtualHostWithInterceptors, error) { + vhs := make([]VirtualHostWithInterceptors, len(config.VirtualHosts)) + for _, vh := range config.VirtualHosts { + vhwi, err := f.convertVirtualHost(vh) + if err != nil { + return nil, fmt.Errorf("virtual host construction: %v", err) + } + vhs = append(vhs, vhwi) + } + return vhs, nil +} + +func (f *FilterChain) convertVirtualHost(virtualHost *VirtualHost) (VirtualHostWithInterceptors, error) { + rs := make([]RouteWithInterceptors, len(virtualHost.Routes)) + for i, r := range virtualHost.Routes { + var err error + rs[i].ActionType = r.ActionType + rs[i].M, err = RouteToMatcher(r) + if err != nil { + return VirtualHostWithInterceptors{}, fmt.Errorf("matcher construction: %v", err) + } + for _, filter := range f.HTTPFilters { + // Route is highest priority on server side, as there is no concept + // of an upstream cluster on server side. + override := r.HTTPFilterConfigOverride[filter.Name] + if override == nil { + // Virtual Host is second priority. + override = virtualHost.HTTPFilterConfigOverride[filter.Name] + } + sb, ok := filter.Filter.(httpfilter.ServerInterceptorBuilder) + if !ok { + // Should not happen if it passed xdsClient validation. + return VirtualHostWithInterceptors{}, fmt.Errorf("filter does not support use in server") + } + si, err := sb.BuildServerInterceptor(filter.Config, override) + if err != nil { + return VirtualHostWithInterceptors{}, fmt.Errorf("filter construction: %v", err) + } + if si != nil { + rs[i].Interceptors = append(rs[i].Interceptors, si) + } + } + } + return VirtualHostWithInterceptors{Domains: virtualHost.Domains, Routes: rs}, nil +} + +// SourceType specifies the connection source IP match type. +type SourceType int + +const ( + // SourceTypeAny matches connection attempts from any source. + SourceTypeAny SourceType = iota + // SourceTypeSameOrLoopback matches connection attempts from the same host. + SourceTypeSameOrLoopback + // SourceTypeExternal matches connection attempts from a different host. + SourceTypeExternal +) + +// FilterChainManager contains all the match criteria specified through all +// filter chains in a single Listener resource. It also contains the default +// filter chain specified in the Listener resource. It provides two important +// pieces of functionality: +// 1. Validate the filter chains in an incoming Listener resource to make sure +// that there aren't filter chains which contain the same match criteria. +// 2. As part of performing the above validation, it builds an internal data +// structure which will if used to look up the matching filter chain at +// connection time. +// +// The logic specified in the documentation around the xDS FilterChainMatch +// proto mentions 8 criteria to match on. +// The following order applies: +// +// 1. Destination port. +// 2. Destination IP address. +// 3. Server name (e.g. SNI for TLS protocol), +// 4. Transport protocol. +// 5. Application protocols (e.g. ALPN for TLS protocol). +// 6. Source type (e.g. any, local or external network). +// 7. Source IP address. +// 8. Source port. +type FilterChainManager struct { + logger *grpclog.PrefixLogger + // Destination prefix is the first match criteria that we support. + // Therefore, this multi-stage map is indexed on destination prefixes + // specified in the match criteria. + // Unspecified destination prefix matches end up as a wildcard entry here + // with a key of 0.0.0.0/0. + dstPrefixMap map[string]*destPrefixEntry + + // At connection time, we do not have the actual destination prefix to match + // on. We only have the real destination address of the incoming connection. + // This means that we cannot use the above map at connection time. This list + // contains the map entries from the above map that we can use at connection + // time to find matching destination prefixes in O(n) time. + // + // TODO: Implement LC-trie to support logarithmic time lookups. If that + // involves too much time/effort, sort this slice based on the netmask size. + dstPrefixes []*destPrefixEntry + + def *FilterChain // Default filter chain, if specified. + + // RouteConfigNames are the route configuration names which need to be + // dynamically queried for RDS Configuration for any FilterChains which + // specify to load RDS Configuration dynamically. + RouteConfigNames map[string]bool +} + +// destPrefixEntry is the value type of the map indexed on destination prefixes. +type destPrefixEntry struct { + // The actual destination prefix. Set to nil for unspecified prefixes. + net *net.IPNet + // We need to keep track of the transport protocols seen as part of the + // config validation (and internal structure building) phase. The only two + // values that we support are empty string and "raw_buffer", with the latter + // taking preference. Once we have seen one filter chain with "raw_buffer", + // we can drop everything other filter chain with an empty transport + // protocol. + rawBufferSeen bool + // For each specified source type in the filter chain match criteria, this + // array points to the set of specified source prefixes. + // Unspecified source type matches end up as a wildcard entry here with an + // index of 0, which actually represents the source type `ANY`. + srcTypeArr sourceTypesArray +} + +// An array for the fixed number of source types that we have. +type sourceTypesArray [3]*sourcePrefixes + +// sourcePrefixes contains source prefix related information specified in the +// match criteria. These are pointed to by the array of source types. +type sourcePrefixes struct { + // These are very similar to the 'dstPrefixMap' and 'dstPrefixes' field of + // FilterChainManager. Go there for more info. + srcPrefixMap map[string]*sourcePrefixEntry + srcPrefixes []*sourcePrefixEntry +} + +// sourcePrefixEntry contains match criteria per source prefix. +type sourcePrefixEntry struct { + // The actual destination prefix. Set to nil for unspecified prefixes. + net *net.IPNet + // Mapping from source ports specified in the match criteria to the actual + // filter chain. Unspecified source port matches en up as a wildcard entry + // here with a key of 0. + srcPortMap map[int]*FilterChain +} + +// NewFilterChainManager parses the received Listener resource and builds a +// FilterChainManager. Returns a non-nil error on validation failures. +// +// This function is only exported so that tests outside of this package can +// create a FilterChainManager. +func NewFilterChainManager(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger) (*FilterChainManager, error) { + // Parse all the filter chains and build the internal data structures. + fci := &FilterChainManager{ + logger: logger, + dstPrefixMap: make(map[string]*destPrefixEntry), + RouteConfigNames: make(map[string]bool), + } + if err := fci.addFilterChains(lis.GetFilterChains()); err != nil { + return nil, err + } + // Build the source and dest prefix slices used by Lookup(). + fcSeen := false + for _, dstPrefix := range fci.dstPrefixMap { + fci.dstPrefixes = append(fci.dstPrefixes, dstPrefix) + for _, st := range dstPrefix.srcTypeArr { + if st == nil { + continue + } + for _, srcPrefix := range st.srcPrefixMap { + st.srcPrefixes = append(st.srcPrefixes, srcPrefix) + for _, fc := range srcPrefix.srcPortMap { + if fc != nil { + fcSeen = true + } + } + } + } + } + + // Retrieve the default filter chain. The match criteria specified on the + // default filter chain is never used. The default filter chain simply gets + // used when none of the other filter chains match. + var def *FilterChain + if dfc := lis.GetDefaultFilterChain(); dfc != nil { + var err error + if def, err = fci.filterChainFromProto(dfc); err != nil { + return nil, err + } + } + fci.def = def + + // If there are no supported filter chains and no default filter chain, we + // fail here. This will call the Listener resource to be NACK'ed. + if !fcSeen && fci.def == nil { + return nil, fmt.Errorf("no supported filter chains and no default filter chain") + } + return fci, nil +} + +// addFilterChains parses the filter chains in fcs and adds the required +// internal data structures corresponding to the match criteria. +func (fci *FilterChainManager) addFilterChains(fcs []*v3listenerpb.FilterChain) error { + for _, fc := range fcs { + fcm := fc.GetFilterChainMatch() + if fcm.GetDestinationPort().GetValue() != 0 { + // Destination port is the first match criteria and we do not + // support filter chains which contains this match criteria. + fci.logger.Warningf("Dropping filter chain %+v since it contains unsupported destination_port match field", fc) + continue + } + + // Build the internal representation of the filter chain match fields. + if err := fci.addFilterChainsForDestPrefixes(fc); err != nil { + return err + } + } + + return nil +} + +func (fci *FilterChainManager) addFilterChainsForDestPrefixes(fc *v3listenerpb.FilterChain) error { + ranges := fc.GetFilterChainMatch().GetPrefixRanges() + dstPrefixes := make([]*net.IPNet, 0, len(ranges)) + for _, pr := range ranges { + cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue()) + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse destination prefix range: %+v", pr) + } + dstPrefixes = append(dstPrefixes, ipnet) + } + + if len(dstPrefixes) == 0 { + // Use the unspecified entry when destination prefix is unspecified, and + // set the `net` field to nil. + if fci.dstPrefixMap[unspecifiedPrefixMapKey] == nil { + fci.dstPrefixMap[unspecifiedPrefixMapKey] = &destPrefixEntry{} + } + return fci.addFilterChainsForServerNames(fci.dstPrefixMap[unspecifiedPrefixMapKey], fc) + } + for _, prefix := range dstPrefixes { + p := prefix.String() + if fci.dstPrefixMap[p] == nil { + fci.dstPrefixMap[p] = &destPrefixEntry{net: prefix} + } + if err := fci.addFilterChainsForServerNames(fci.dstPrefixMap[p], fc); err != nil { + return err + } + } + return nil +} + +func (fci *FilterChainManager) addFilterChainsForServerNames(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + // Filter chains specifying server names in their match criteria always fail + // a match at connection time. So, these filter chains can be dropped now. + if len(fc.GetFilterChainMatch().GetServerNames()) != 0 { + fci.logger.Warningf("Dropping filter chain %+v since it contains unsupported server_names match field", fc) + return nil + } + + return fci.addFilterChainsForTransportProtocols(dstEntry, fc) +} + +func (fci *FilterChainManager) addFilterChainsForTransportProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + tp := fc.GetFilterChainMatch().GetTransportProtocol() + switch { + case tp != "" && tp != "raw_buffer": + // Only allow filter chains with transport protocol set to empty string + // or "raw_buffer". + fci.logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc) + return nil + case tp == "" && dstEntry.rawBufferSeen: + // If we have already seen filter chains with transport protocol set to + // "raw_buffer", we can drop filter chains with transport protocol set + // to empty string, since the former takes precedence. + fci.logger.Warningf("Dropping filter chain %+v since it contains unsupported value for transport_protocols match field", fc) + return nil + case tp != "" && !dstEntry.rawBufferSeen: + // This is the first "raw_buffer" that we are seeing. Set the bit and + // reset the source types array which might contain entries for filter + // chains with transport protocol set to empty string. + dstEntry.rawBufferSeen = true + dstEntry.srcTypeArr = sourceTypesArray{} + } + return fci.addFilterChainsForApplicationProtocols(dstEntry, fc) +} + +func (fci *FilterChainManager) addFilterChainsForApplicationProtocols(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + if len(fc.GetFilterChainMatch().GetApplicationProtocols()) != 0 { + fci.logger.Warningf("Dropping filter chain %+v since it contains unsupported application_protocols match field", fc) + return nil + } + return fci.addFilterChainsForSourceType(dstEntry, fc) +} + +// addFilterChainsForSourceType adds source types to the internal data +// structures and delegates control to addFilterChainsForSourcePrefixes to +// continue building the internal data structure. +func (fci *FilterChainManager) addFilterChainsForSourceType(dstEntry *destPrefixEntry, fc *v3listenerpb.FilterChain) error { + var srcType SourceType + switch st := fc.GetFilterChainMatch().GetSourceType(); st { + case v3listenerpb.FilterChainMatch_ANY: + srcType = SourceTypeAny + case v3listenerpb.FilterChainMatch_SAME_IP_OR_LOOPBACK: + srcType = SourceTypeSameOrLoopback + case v3listenerpb.FilterChainMatch_EXTERNAL: + srcType = SourceTypeExternal + default: + return fmt.Errorf("unsupported source type: %v", st) + } + + st := int(srcType) + if dstEntry.srcTypeArr[st] == nil { + dstEntry.srcTypeArr[st] = &sourcePrefixes{srcPrefixMap: make(map[string]*sourcePrefixEntry)} + } + return fci.addFilterChainsForSourcePrefixes(dstEntry.srcTypeArr[st].srcPrefixMap, fc) +} + +// addFilterChainsForSourcePrefixes adds source prefixes to the internal data +// structures and delegates control to addFilterChainsForSourcePorts to continue +// building the internal data structure. +func (fci *FilterChainManager) addFilterChainsForSourcePrefixes(srcPrefixMap map[string]*sourcePrefixEntry, fc *v3listenerpb.FilterChain) error { + ranges := fc.GetFilterChainMatch().GetSourcePrefixRanges() + srcPrefixes := make([]*net.IPNet, 0, len(ranges)) + for _, pr := range fc.GetFilterChainMatch().GetSourcePrefixRanges() { + cidr := fmt.Sprintf("%s/%d", pr.GetAddressPrefix(), pr.GetPrefixLen().GetValue()) + _, ipnet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("failed to parse source prefix range: %+v", pr) + } + srcPrefixes = append(srcPrefixes, ipnet) + } + + if len(srcPrefixes) == 0 { + // Use the unspecified entry when destination prefix is unspecified, and + // set the `net` field to nil. + if srcPrefixMap[unspecifiedPrefixMapKey] == nil { + srcPrefixMap[unspecifiedPrefixMapKey] = &sourcePrefixEntry{ + srcPortMap: make(map[int]*FilterChain), + } + } + return fci.addFilterChainsForSourcePorts(srcPrefixMap[unspecifiedPrefixMapKey], fc) + } + for _, prefix := range srcPrefixes { + p := prefix.String() + if srcPrefixMap[p] == nil { + srcPrefixMap[p] = &sourcePrefixEntry{ + net: prefix, + srcPortMap: make(map[int]*FilterChain), + } + } + if err := fci.addFilterChainsForSourcePorts(srcPrefixMap[p], fc); err != nil { + return err + } + } + return nil +} + +// addFilterChainsForSourcePorts adds source ports to the internal data +// structures and completes the process of building the internal data structure. +// It is here that we determine if there are multiple filter chains with +// overlapping matching rules. +func (fci *FilterChainManager) addFilterChainsForSourcePorts(srcEntry *sourcePrefixEntry, fcProto *v3listenerpb.FilterChain) error { + ports := fcProto.GetFilterChainMatch().GetSourcePorts() + srcPorts := make([]int, 0, len(ports)) + for _, port := range ports { + srcPorts = append(srcPorts, int(port)) + } + + fc, err := fci.filterChainFromProto(fcProto) + if err != nil { + return err + } + + if len(srcPorts) == 0 { + // Use the wildcard port '0', when source ports are unspecified. + if curFC := srcEntry.srcPortMap[0]; curFC != nil { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + srcEntry.srcPortMap[0] = fc + return nil + } + for _, port := range srcPorts { + if curFC := srcEntry.srcPortMap[port]; curFC != nil { + return errors.New("multiple filter chains with overlapping matching rules are defined") + } + srcEntry.srcPortMap[port] = fc + } + return nil +} + +// filterChainFromProto extracts the relevant information from the FilterChain +// proto and stores it in our internal representation. It also persists any +// RouteNames which need to be queried dynamically via RDS. +func (fci *FilterChainManager) filterChainFromProto(fc *v3listenerpb.FilterChain) (*FilterChain, error) { + filterChain, err := processNetworkFilters(fc.GetFilters()) + if err != nil { + return nil, err + } + // These route names will be dynamically queried via RDS in the wrapped + // listener, which receives the LDS response, if specified for the filter + // chain. + if filterChain.RouteConfigName != "" { + fci.RouteConfigNames[filterChain.RouteConfigName] = true + } + // If the transport_socket field is not specified, it means that the control + // plane has not sent us any security config. This is fine and the server + // will use the fallback credentials configured as part of the + // xdsCredentials. + ts := fc.GetTransportSocket() + if ts == nil { + return filterChain, nil + } + if name := ts.GetName(); name != transportSocketName { + return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + any := ts.GetTypedConfig() + if any == nil || any.TypeUrl != version.V3DownstreamTLSContextURL { + return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl) + } + downstreamCtx := &v3tlspb.DownstreamTlsContext{} + if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil { + return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err) + } + if downstreamCtx.GetRequireSni().GetValue() { + return nil, fmt.Errorf("require_sni field set to true in DownstreamTlsContext message: %v", downstreamCtx) + } + if downstreamCtx.GetOcspStaplePolicy() != v3tlspb.DownstreamTlsContext_LENIENT_STAPLING { + return nil, fmt.Errorf("ocsp_staple_policy field set to unsupported value in DownstreamTlsContext message: %v", downstreamCtx) + } + // The following fields from `DownstreamTlsContext` are ignore: + // - disable_stateless_session_resumption + // - session_ticket_keys + // - session_ticket_keys_sds_secret_config + // - session_timeout + if downstreamCtx.GetCommonTlsContext() == nil { + return nil, errors.New("DownstreamTlsContext in LDS response does not contain a CommonTlsContext") + } + sc, err := securityConfigFromCommonTLSContext(downstreamCtx.GetCommonTlsContext(), true) + if err != nil { + return nil, err + } + if sc == nil { + // sc == nil is a valid case where the control plane has not sent us any + // security configuration. xDS creds will use fallback creds. + return filterChain, nil + } + sc.RequireClientCert = downstreamCtx.GetRequireClientCertificate().GetValue() + if sc.RequireClientCert && sc.RootInstanceName == "" { + return nil, errors.New("security configuration on the server-side does not contain root certificate provider instance name, but require_client_cert field is set") + } + filterChain.SecurityCfg = sc + return filterChain, nil +} + +// Validate takes a function to validate the FilterChains in this manager. +func (fci *FilterChainManager) Validate(f func(fc *FilterChain) error) error { + for _, dst := range fci.dstPrefixMap { + for _, srcType := range dst.srcTypeArr { + if srcType == nil { + continue + } + for _, src := range srcType.srcPrefixMap { + for _, fc := range src.srcPortMap { + if err := f(fc); err != nil { + return err + } + } + } + } + } + return f(fci.def) +} + +func processNetworkFilters(filters []*v3listenerpb.Filter) (*FilterChain, error) { + filterChain := &FilterChain{} + seenNames := make(map[string]bool, len(filters)) + seenHCM := false + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, fmt.Errorf("network filters {%+v} is missing name field in filter: {%+v}", filters, filter) + } + if seenNames[name] { + return nil, fmt.Errorf("network filters {%+v} has duplicate filter name %q", filters, name) + } + seenNames[name] = true + + // Network filters have a oneof field named `config_type` where we + // only support `TypedConfig` variant. + switch typ := filter.GetConfigType().(type) { + case *v3listenerpb.Filter_TypedConfig: + // The typed_config field has an `anypb.Any` proto which could + // directly contain the serialized bytes of the actual filter + // configuration, or it could be encoded as a `TypedStruct`. + // TODO: Add support for `TypedStruct`. + tc := filter.GetTypedConfig() + + // The only network filter that we currently support is the v3 + // HttpConnectionManager. So, we can directly check the type_url + // and unmarshal the config. + // TODO: Implement a registry of supported network filters (like + // we have for HTTP filters), when we have to support network + // filters other than HttpConnectionManager. + if tc.GetTypeUrl() != version.V3HTTPConnManagerURL { + return nil, fmt.Errorf("network filters {%+v} has unsupported network filter %q in filter {%+v}", filters, tc.GetTypeUrl(), filter) + } + hcm := &v3httppb.HttpConnectionManager{} + if err := ptypes.UnmarshalAny(tc, hcm); err != nil { + return nil, fmt.Errorf("network filters {%+v} failed unmarshaling of network filter {%+v}: %v", filters, filter, err) + } + // "Any filters after HttpConnectionManager should be ignored during + // connection processing but still be considered for validity. + // HTTPConnectionManager must have valid http_filters." - A36 + filters, err := processHTTPFilters(hcm.GetHttpFilters(), true) + if err != nil { + return nil, fmt.Errorf("network filters {%+v} had invalid server side HTTP Filters {%+v}: %v", filters, hcm.GetHttpFilters(), err) + } + if !seenHCM { + // Validate for RBAC in only the HCM that will be used, since this isn't a logical validation failure, + // it's simply a validation to support RBAC HTTP Filter. + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if hcm.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", hcm) + } + if len(hcm.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", hcm) + } + + // TODO: Implement terminal filter logic, as per A36. + filterChain.HTTPFilters = filters + seenHCM = true + if !envconfig.XDSRBAC { + continue + } + switch hcm.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if hcm.GetRds().GetConfigSource().GetAds() == nil { + return nil, fmt.Errorf("ConfigSource is not ADS: %+v", hcm) + } + name := hcm.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", hcm) + } + filterChain.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + // "RouteConfiguration validation logic inherits all + // previous validations made for client-side usage as RDS + // does not distinguish between client-side and + // server-side." - A36 + // Can specify v3 here, as will never get to this function + // if v2. + routeU, err := generateRDSUpdateFromRouteConfiguration(hcm.GetRouteConfig(), nil, false) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + filterChain.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", hcm) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", hcm.RouteSpecifier) + } + } + default: + return nil, fmt.Errorf("network filters {%+v} has unsupported config_type %T in filter %s", filters, typ, filter.GetName()) + } + } + if !seenHCM { + return nil, fmt.Errorf("network filters {%+v} missing HttpConnectionManager filter", filters) + } + return filterChain, nil +} + +// FilterChainLookupParams wraps parameters to be passed to Lookup. +type FilterChainLookupParams struct { + // IsUnspecified indicates whether the server is listening on a wildcard + // address, "0.0.0.0" for IPv4 and "::" for IPv6. Only when this is set to + // true, do we consider the destination prefixes specified in the filter + // chain match criteria. + IsUnspecifiedListener bool + // DestAddr is the local address of an incoming connection. + DestAddr net.IP + // SourceAddr is the remote address of an incoming connection. + SourceAddr net.IP + // SourcePort is the remote port of an incoming connection. + SourcePort int +} + +// Lookup returns the most specific matching filter chain to be used for an +// incoming connection on the server side. +// +// Returns a non-nil error if no matching filter chain could be found or +// multiple matching filter chains were found, and in both cases, the incoming +// connection must be dropped. +func (fci *FilterChainManager) Lookup(params FilterChainLookupParams) (*FilterChain, error) { + dstPrefixes := filterByDestinationPrefixes(fci.dstPrefixes, params.IsUnspecifiedListener, params.DestAddr) + if len(dstPrefixes) == 0 { + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain based on destination prefix match for %+v", params) + } + + srcType := SourceTypeExternal + if params.SourceAddr.Equal(params.DestAddr) || params.SourceAddr.IsLoopback() { + srcType = SourceTypeSameOrLoopback + } + srcPrefixes := filterBySourceType(dstPrefixes, srcType) + if len(srcPrefixes) == 0 { + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain based on source type match for %+v", params) + } + srcPrefixEntry, err := filterBySourcePrefixes(srcPrefixes, params.SourceAddr) + if err != nil { + return nil, err + } + if fc := filterBySourcePorts(srcPrefixEntry, params.SourcePort); fc != nil { + return fc, nil + } + if fci.def != nil { + return fci.def, nil + } + return nil, fmt.Errorf("no matching filter chain after all match criteria for %+v", params) +} + +// filterByDestinationPrefixes is the first stage of the filter chain +// matching algorithm. It takes the complete set of configured filter chain +// matchers and returns the most specific matchers based on the destination +// prefix match criteria (the prefixes which match the most number of bits). +func filterByDestinationPrefixes(dstPrefixes []*destPrefixEntry, isUnspecified bool, dstAddr net.IP) []*destPrefixEntry { + if !isUnspecified { + // Destination prefix matchers are considered only when the listener is + // bound to the wildcard address. + return dstPrefixes + } + + var matchingDstPrefixes []*destPrefixEntry + maxSubnetMatch := noPrefixMatch + for _, prefix := range dstPrefixes { + if prefix.net != nil && !prefix.net.Contains(dstAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real net.IPNet + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if prefix.net != nil { + matchSize, _ = prefix.net.Mask.Size() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingDstPrefixes = make([]*destPrefixEntry, 0, 1) + } + matchingDstPrefixes = append(matchingDstPrefixes, prefix) + } + return matchingDstPrefixes +} + +// filterBySourceType is the second stage of the matching algorithm. It +// trims the filter chains based on the most specific source type match. +func filterBySourceType(dstPrefixes []*destPrefixEntry, srcType SourceType) []*sourcePrefixes { + var ( + srcPrefixes []*sourcePrefixes + bestSrcTypeMatch int + ) + for _, prefix := range dstPrefixes { + var ( + srcPrefix *sourcePrefixes + match int + ) + switch srcType { + case SourceTypeExternal: + match = int(SourceTypeExternal) + srcPrefix = prefix.srcTypeArr[match] + case SourceTypeSameOrLoopback: + match = int(SourceTypeSameOrLoopback) + srcPrefix = prefix.srcTypeArr[match] + } + if srcPrefix == nil { + match = int(SourceTypeAny) + srcPrefix = prefix.srcTypeArr[match] + } + if match < bestSrcTypeMatch { + continue + } + if match > bestSrcTypeMatch { + bestSrcTypeMatch = match + srcPrefixes = make([]*sourcePrefixes, 0) + } + if srcPrefix != nil { + // The source type array always has 3 entries, but these could be + // nil if the appropriate source type match was not specified. + srcPrefixes = append(srcPrefixes, srcPrefix) + } + } + return srcPrefixes +} + +// filterBySourcePrefixes is the third stage of the filter chain matching +// algorithm. It trims the filter chains based on the source prefix. At most one +// filter chain with the most specific match progress to the next stage. +func filterBySourcePrefixes(srcPrefixes []*sourcePrefixes, srcAddr net.IP) (*sourcePrefixEntry, error) { + var matchingSrcPrefixes []*sourcePrefixEntry + maxSubnetMatch := noPrefixMatch + for _, sp := range srcPrefixes { + for _, prefix := range sp.srcPrefixes { + if prefix.net != nil && !prefix.net.Contains(srcAddr) { + // Skip prefixes which don't match. + continue + } + // For unspecified prefixes, since we do not store a real net.IPNet + // inside prefix, we do not perform a match. Instead we simply set + // the matchSize to -1, which is less than the matchSize (0) for a + // wildcard prefix. + matchSize := unspecifiedPrefixMatch + if prefix.net != nil { + matchSize, _ = prefix.net.Mask.Size() + } + if matchSize < maxSubnetMatch { + continue + } + if matchSize > maxSubnetMatch { + maxSubnetMatch = matchSize + matchingSrcPrefixes = make([]*sourcePrefixEntry, 0, 1) + } + matchingSrcPrefixes = append(matchingSrcPrefixes, prefix) + } + } + if len(matchingSrcPrefixes) == 0 { + // Finding no match is not an error condition. The caller will end up + // using the default filter chain if one was configured. + return nil, nil + } + // We expect at most a single matching source prefix entry at this point. If + // we have multiple entries here, and some of their source port matchers had + // wildcard entries, we could be left with more than one matching filter + // chain and hence would have been flagged as an invalid configuration at + // config validation time. + if len(matchingSrcPrefixes) != 1 { + return nil, errors.New("multiple matching filter chains") + } + return matchingSrcPrefixes[0], nil +} + +// filterBySourcePorts is the last stage of the filter chain matching +// algorithm. It trims the filter chains based on the source ports. +func filterBySourcePorts(spe *sourcePrefixEntry, srcPort int) *FilterChain { + if spe == nil { + return nil + } + // A match could be a wildcard match (this happens when the match + // criteria does not specify source ports) or a specific port match (this + // happens when the match criteria specifies a set of ports and the source + // port of the incoming connection matches one of the specified ports). The + // latter is considered to be a more specific match. + if fc := spe.srcPortMap[srcPort]; fc != nil { + return fc + } + if fc := spe.srcPortMap[0]; fc != nil { + return fc + } + return nil +} diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go new file mode 100644 index 0000000000..9e720da1d5 --- /dev/null +++ b/xds/client/resource/locality_id.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package internal contains functions/structs shared by xds +// balancers/resolvers. +package resource + +import ( + "encoding/json" + "fmt" + + "google.golang.org/grpc/resolver" +) + +// LocalityID is xds.Locality without XXX fields, so it can be used as map +// keys. +// +// xds.Locality cannot be map keys because one of the XXX fields is a slice. +type LocalityID struct { + Region string `json:"region,omitempty"` + Zone string `json:"zone,omitempty"` + SubZone string `json:"subZone,omitempty"` +} + +// ToString generates a string representation of LocalityID by marshalling it into +// json. Not calling it String() so printf won't call it. +func (l LocalityID) ToString() (string, error) { + b, err := json.Marshal(l) + if err != nil { + return "", err + } + return string(b), nil +} + +// Equal allows the values to be compared by Attributes.Equal. +func (l LocalityID) Equal(o interface{}) bool { + ol, ok := o.(LocalityID) + if !ok { + return false + } + return l.Region == ol.Region && l.Zone == ol.Zone && l.SubZone == ol.SubZone +} + +// LocalityIDFromString converts a json representation of locality, into a +// LocalityID struct. +func LocalityIDFromString(s string) (ret LocalityID, _ error) { + err := json.Unmarshal([]byte(s), &ret) + if err != nil { + return LocalityID{}, fmt.Errorf("%s is not a well formatted locality ID, error: %v", s, err) + } + return ret, nil +} + +type localityKeyType string + +const localityKey = localityKeyType("grpc.xds.internal.address.locality") + +// GetLocalityID returns the locality ID of addr. +func GetLocalityID(addr resolver.Address) LocalityID { + path, _ := addr.BalancerAttributes.Value(localityKey).(LocalityID) + return path +} + +// SetLocalityID sets locality ID in addr to l. +func SetLocalityID(addr resolver.Address, l LocalityID) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(localityKey, l) + return addr +} diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go new file mode 100644 index 0000000000..163b3035b9 --- /dev/null +++ b/xds/client/resource/matcher.go @@ -0,0 +1,275 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "fmt" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + metedatautils "dubbo.apache.org/dubbo-go/v3/xds/utils/metadata" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "google.golang.org/grpc/metadata" +) + +// RouteToMatcher converts a route to a Matcher to match incoming RPC's against. +func RouteToMatcher(r *Route) (*CompositeMatcher, error) { + var pm pathMatcher + switch { + case r.Regex != nil: + pm = newPathRegexMatcher(r.Regex) + case r.Path != nil: + pm = newPathExactMatcher(*r.Path, r.CaseInsensitive) + case r.Prefix != nil: + pm = newPathPrefixMatcher(*r.Prefix, r.CaseInsensitive) + default: + return nil, fmt.Errorf("illegal route: missing path_matcher") + } + + headerMatchers := make([]matcher.HeaderMatcher, 0, len(r.Headers)) + for _, h := range r.Headers { + var matcherT matcher.HeaderMatcher + invert := h.InvertMatch != nil && *h.InvertMatch + switch { + case h.ExactMatch != nil && *h.ExactMatch != "": + matcherT = matcher.NewHeaderExactMatcher(h.Name, *h.ExactMatch, invert) + case h.RegexMatch != nil: + matcherT = matcher.NewHeaderRegexMatcher(h.Name, h.RegexMatch, invert) + case h.PrefixMatch != nil && *h.PrefixMatch != "": + matcherT = matcher.NewHeaderPrefixMatcher(h.Name, *h.PrefixMatch, invert) + case h.SuffixMatch != nil && *h.SuffixMatch != "": + matcherT = matcher.NewHeaderSuffixMatcher(h.Name, *h.SuffixMatch, invert) + case h.RangeMatch != nil: + matcherT = matcher.NewHeaderRangeMatcher(h.Name, h.RangeMatch.Start, h.RangeMatch.End, invert) + case h.PresentMatch != nil: + matcherT = matcher.NewHeaderPresentMatcher(h.Name, *h.PresentMatch, invert) + default: + return nil, fmt.Errorf("illegal route: missing header_match_specifier") + } + headerMatchers = append(headerMatchers, matcherT) + } + + var fractionMatcher *fractionMatcher + if r.Fraction != nil { + fractionMatcher = newFractionMatcher(*r.Fraction) + } + return newCompositeMatcher(pm, headerMatchers, fractionMatcher), nil +} + +// CompositeMatcher is a matcher that holds onto many matchers and aggregates +// the matching results. +type CompositeMatcher struct { + pm pathMatcher + hms []matcher.HeaderMatcher + fm *fractionMatcher +} + +func newCompositeMatcher(pm pathMatcher, hms []matcher.HeaderMatcher, fm *fractionMatcher) *CompositeMatcher { + return &CompositeMatcher{pm: pm, hms: hms, fm: fm} +} + +// Match returns true if all matchers return true. +func (a *CompositeMatcher) Match(info iresolver.RPCInfo) bool { + if a.pm != nil && !a.pm.match(info.Method) { + return false + } + + // Call headerMatchers even if md is nil, because routes may match + // non-presence of some headers. + var md metadata.MD + if info.Context != nil { + md, _ = metadata.FromOutgoingContext(info.Context) + if extraMD, ok := metedatautils.ExtraMetadata(info.Context); ok { + md = metadata.Join(md, extraMD) + // Remove all binary headers. They are hard to match with. May need + // to add back if asked by users. + for k := range md { + if strings.HasSuffix(k, "-bin") { + delete(md, k) + } + } + } + } + for _, m := range a.hms { + if !m.Match(md) { + return false + } + } + + if a.fm != nil && !a.fm.match() { + return false + } + return true +} + +func (a *CompositeMatcher) String() string { + var ret string + if a.pm != nil { + ret += a.pm.String() + } + for _, m := range a.hms { + ret += m.String() + } + if a.fm != nil { + ret += a.fm.String() + } + return ret +} + +type fractionMatcher struct { + fraction int64 // real fraction is fraction/1,000,000. +} + +func newFractionMatcher(fraction uint32) *fractionMatcher { + return &fractionMatcher{fraction: int64(fraction)} +} + +// RandInt63n overwrites grpcrand for control in tests. +var RandInt63n = grpcrand.Int63n + +func (fm *fractionMatcher) match() bool { + t := RandInt63n(1000000) + return t <= fm.fraction +} + +func (fm *fractionMatcher) String() string { + return fmt.Sprintf("fraction:%v", fm.fraction) +} + +type domainMatchType int + +const ( + domainMatchTypeInvalid domainMatchType = iota + domainMatchTypeUniversal + domainMatchTypePrefix + domainMatchTypeSuffix + domainMatchTypeExact +) + +// Exact > Suffix > Prefix > Universal > Invalid. +func (t domainMatchType) betterThan(b domainMatchType) bool { + return t > b +} + +func matchTypeForDomain(d string) domainMatchType { + if d == "" { + return domainMatchTypeInvalid + } + if d == "*" { + return domainMatchTypeUniversal + } + if strings.HasPrefix(d, "*") { + return domainMatchTypeSuffix + } + if strings.HasSuffix(d, "*") { + return domainMatchTypePrefix + } + if strings.Contains(d, "*") { + return domainMatchTypeInvalid + } + return domainMatchTypeExact +} + +func match(domain, host string) (domainMatchType, bool) { + switch typ := matchTypeForDomain(domain); typ { + case domainMatchTypeInvalid: + return typ, false + case domainMatchTypeUniversal: + return typ, true + case domainMatchTypePrefix: + // abc.* + return typ, strings.HasPrefix(host, strings.TrimSuffix(domain, "*")) + case domainMatchTypeSuffix: + // *.123 + return typ, strings.HasSuffix(host, strings.TrimPrefix(domain, "*")) + case domainMatchTypeExact: + return typ, domain == host + default: + return domainMatchTypeInvalid, false + } +} + +// FindBestMatchingVirtualHost returns the virtual host whose domains field best +// matches host +// +// The domains field support 4 different matching pattern types: +// - Exact match +// - Suffix match (e.g. “*ABC”) +// - Prefix match (e.g. “ABC*) +// - Universal match (e.g. “*”) +// +// The best match is defined as: +// - A match is better if it’s matching pattern type is better +// - Exact match > suffix match > prefix match > universal match +// - If two matches are of the same pattern type, the longer match is better +// - This is to compare the length of the matching pattern, e.g. “*ABCDE” > +// “*ABC” +func FindBestMatchingVirtualHost(host string, vHosts []*VirtualHost) *VirtualHost { // Maybe move this crap to client + var ( + matchVh *VirtualHost + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.Domains { + typ, matched := match(domain, host) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} + +// FindBestMatchingVirtualHostServer returns the virtual host whose domains field best +// matches authority. +func FindBestMatchingVirtualHostServer(authority string, vHosts []VirtualHostWithInterceptors) *VirtualHostWithInterceptors { + var ( + matchVh *VirtualHostWithInterceptors + matchType = domainMatchTypeInvalid + matchLen int + ) + for _, vh := range vHosts { + for _, domain := range vh.Domains { + typ, matched := match(domain, authority) + if typ == domainMatchTypeInvalid { + // The rds response is invalid. + return nil + } + if matchType.betterThan(typ) || matchType == typ && matchLen >= len(domain) || !matched { + // The previous match has better type, or the previous match has + // better length, or this domain isn't a match. + continue + } + matchVh = &vh + matchType = typ + matchLen = len(domain) + } + } + return matchVh +} diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go new file mode 100644 index 0000000000..86d7253999 --- /dev/null +++ b/xds/client/resource/matcher_path.go @@ -0,0 +1,102 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "regexp" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + +type pathMatcher interface { + match(path string) bool + String() string +} + +type pathExactMatcher struct { + // fullPath is all upper case if caseInsensitive is true. + fullPath string + caseInsensitive bool +} + +func newPathExactMatcher(p string, caseInsensitive bool) *pathExactMatcher { + ret := &pathExactMatcher{ + fullPath: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.fullPath = strings.ToUpper(p) + } + return ret +} + +func (pem *pathExactMatcher) match(path string) bool { + if pem.caseInsensitive { + return pem.fullPath == strings.ToUpper(path) + } + return pem.fullPath == path +} + +func (pem *pathExactMatcher) String() string { + return "pathExact:" + pem.fullPath +} + +type pathPrefixMatcher struct { + // prefix is all upper case if caseInsensitive is true. + prefix string + caseInsensitive bool +} + +func newPathPrefixMatcher(p string, caseInsensitive bool) *pathPrefixMatcher { + ret := &pathPrefixMatcher{ + prefix: p, + caseInsensitive: caseInsensitive, + } + if caseInsensitive { + ret.prefix = strings.ToUpper(p) + } + return ret +} + +func (ppm *pathPrefixMatcher) match(path string) bool { + if ppm.caseInsensitive { + return strings.HasPrefix(strings.ToUpper(path), ppm.prefix) + } + return strings.HasPrefix(path, ppm.prefix) +} + +func (ppm *pathPrefixMatcher) String() string { + return "pathPrefix:" + ppm.prefix +} + +type pathRegexMatcher struct { + re *regexp.Regexp +} + +func newPathRegexMatcher(re *regexp.Regexp) *pathRegexMatcher { + return &pathRegexMatcher{re: re} +} + +func (prm *pathRegexMatcher) match(path string) bool { + return matcher.FullMatchWithRegex(prm.re, path) +} + +func (prm *pathRegexMatcher) String() string { + return "pathRegex:" + prm.re.String() +} diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go new file mode 100644 index 0000000000..629e8b2397 --- /dev/null +++ b/xds/client/resource/name.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "net/url" + "sort" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" +) + +// Name contains the parsed component of an xDS resource name. +// +// An xDS resource name is in the format of +// xdstp://[{authority}]/{resource type}/{id/*}?{context parameters}{#processing directive,*} +// +// See +// https://github.com/cncf/xds/blob/main/proposals/TP1-xds-transport-next.md#uri-based-xds-resource-names +// for details, and examples. +type Name struct { + Scheme string + Authority string + Type string + ID string + + ContextParams map[string]string + + processingDirective string +} + +// ParseName splits the name and returns a struct representation of the Name. +// +// If the name isn't a valid new-style xDS name, field ID is set to the input. +// Note that this is not an error, because we still support the old-style +// resource names (those not starting with "xdstp:"). +// +// The caller can tell if the parsing is successful by checking the returned +// Scheme. +func ParseName(name string) *Name { + if !envconfig.XDSFederation { + // Return "" scheme to use the default authority for the server. + return &Name{ID: name} + } + if !strings.Contains(name, "://") { + // Only the long form URL, with ://, is valid. + return &Name{ID: name} + } + parsed, err := url.Parse(name) + if err != nil { + return &Name{ID: name} + } + + ret := &Name{ + Scheme: parsed.Scheme, + Authority: parsed.Host, + } + split := strings.SplitN(parsed.Path, "/", 3) + if len(split) < 3 { + // Path is in the format of "/type/id". There must be at least 3 + // segments after splitting. + return &Name{ID: name} + } + ret.Type = split[1] + ret.ID = split[2] + if len(parsed.Query()) != 0 { + ret.ContextParams = make(map[string]string) + for k, vs := range parsed.Query() { + if len(vs) > 0 { + // We only keep one value of each key. Behavior for multiple values + // is undefined. + ret.ContextParams[k] = vs[0] + } + } + } + // TODO: processing directive (the part comes after "#" in the URL, stored + // in parsed.RawFragment) is kept but not processed. Add support for that + // when it's needed. + ret.processingDirective = parsed.RawFragment + return ret +} + +// String returns a canonicalized string of name. The context parameters are +// sorted by the keys. +func (n *Name) String() string { + if n.Scheme == "" { + return n.ID + } + + // Sort and build query. + keys := make([]string, 0, len(n.ContextParams)) + for k := range n.ContextParams { + keys = append(keys, k) + } + sort.Strings(keys) + var pairs []string + for _, k := range keys { + pairs = append(pairs, strings.Join([]string{k, n.ContextParams[k]}, "=")) + } + rawQuery := strings.Join(pairs, "&") + + path := n.Type + if n.ID != "" { + path = path + "/" + n.ID + } + + tempURL := &url.URL{ + Scheme: n.Scheme, + Host: n.Authority, + Path: path, + RawQuery: rawQuery, + RawFragment: n.processingDirective, + } + return tempURL.String() +} diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go new file mode 100644 index 0000000000..262388af8f --- /dev/null +++ b/xds/client/resource/type.go @@ -0,0 +1,150 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "google.golang.org/protobuf/types/known/anypb" +) + +// UpdateValidatorFunc performs validations on update structs using +// context/logic available at the xdsClient layer. Since these validation are +// performed on internal update structs, they can be shared between different +// API clients. +type UpdateValidatorFunc func(interface{}) error + +// UpdateMetadata contains the metadata for each update, including timestamp, +// raw message, and so on. +type UpdateMetadata struct { + // Status is the status of this resource, e.g. ACKed, NACKed, or + // Not_exist(removed). + Status ServiceStatus + // Version is the version of the xds response. Note that this is the version + // of the resource in use (previous ACKed). If a response is NACKed, the + // NACKed version is in ErrState. + Version string + // Timestamp is when the response is received. + Timestamp time.Time + // ErrState is set when the update is NACKed. + ErrState *UpdateErrorMetadata +} + +// IsListenerResource returns true if the provider URL corresponds to an xDS +// Listener resource. +func IsListenerResource(url string) bool { + return url == version.V2ListenerURL || url == version.V3ListenerURL +} + +// IsHTTPConnManagerResource returns true if the provider URL corresponds to an xDS +// HTTPConnManager resource. +func IsHTTPConnManagerResource(url string) bool { + return url == version.V2HTTPConnManagerURL || url == version.V3HTTPConnManagerURL +} + +// IsRouteConfigResource returns true if the provider URL corresponds to an xDS +// RouteConfig resource. +func IsRouteConfigResource(url string) bool { + return url == version.V2RouteConfigURL || url == version.V3RouteConfigURL +} + +// IsClusterResource returns true if the provider URL corresponds to an xDS +// Cluster resource. +func IsClusterResource(url string) bool { + return url == version.V2ClusterURL || url == version.V3ClusterURL +} + +// IsEndpointsResource returns true if the provider URL corresponds to an xDS +// Endpoints resource. +func IsEndpointsResource(url string) bool { + return url == version.V2EndpointsURL || url == version.V3EndpointsURL +} + +// ServiceStatus is the status of the update. +type ServiceStatus int + +const ( + // ServiceStatusUnknown is the default state, before a watch is started for + // the resource. + ServiceStatusUnknown ServiceStatus = iota + // ServiceStatusRequested is when the watch is started, but before and + // response is received. + ServiceStatusRequested + // ServiceStatusNotExist is when the resource doesn't exist in + // state-of-the-world responses (e.g. LDS and CDS), which means the resource + // is removed by the management server. + ServiceStatusNotExist // Resource is removed in the server, in LDS/CDS. + // ServiceStatusACKed is when the resource is ACKed. + ServiceStatusACKed + // ServiceStatusNACKed is when the resource is NACKed. + ServiceStatusNACKed +) + +// UpdateErrorMetadata is part of UpdateMetadata. It contains the error state +// when a response is NACKed. +type UpdateErrorMetadata struct { + // Version is the version of the NACKed response. + Version string + // Err contains why the response was NACKed. + Err error + // Timestamp is when the NACKed response was received. + Timestamp time.Time +} + +// UpdateWithMD contains the raw message of the update and the metadata, +// including version, raw message, timestamp. +// +// This is to be used for config dump and CSDS, not directly by users (like +// resolvers/balancers). +type UpdateWithMD struct { + MD UpdateMetadata + Raw *anypb.Any +} + +// ResourceType identifies resources in a transport protocol agnostic way. These +// will be used in transport version agnostic code, while the versioned API +// clients will map these to appropriate version URLs. +type ResourceType int + +// Version agnostic resource type constants. +const ( + UnknownResource ResourceType = iota + ListenerResource + HTTPConnManagerResource + RouteConfigResource + ClusterResource + EndpointsResource +) + +func (r ResourceType) String() string { + switch r { + case ListenerResource: + return "ListenerResource" + case HTTPConnManagerResource: + return "HTTPConnManagerResource" + case RouteConfigResource: + return "RouteConfigResource" + case ClusterResource: + return "ClusterResource" + case EndpointsResource: + return "EndpointsResource" + default: + return "UnknownResource" + } +} diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go new file mode 100644 index 0000000000..a5fc9fc469 --- /dev/null +++ b/xds/client/resource/type_cds.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import "google.golang.org/protobuf/types/known/anypb" + +// ClusterType is the type of cluster from a received CDS response. +type ClusterType int + +const ( + // ClusterTypeEDS represents the EDS cluster type, which will delegate endpoint + // discovery to the management server. + ClusterTypeEDS ClusterType = iota + // ClusterTypeLogicalDNS represents the Logical DNS cluster type, which essentially + // maps to the gRPC behavior of using the DNS resolver with pick_first LB policy. + ClusterTypeLogicalDNS + // ClusterTypeAggregate represents the Aggregate Cluster type, which provides a + // prioritized list of clusters to use. It is used for failover between clusters + // with a different configuration. + ClusterTypeAggregate +) + +// ClusterLBPolicyRingHash represents ring_hash lb policy, and also contains its +// config. +type ClusterLBPolicyRingHash struct { + MinimumRingSize uint64 + MaximumRingSize uint64 +} + +// ClusterUpdate contains information from a received CDS response, which is of +// interest to the registered CDS watcher. +type ClusterUpdate struct { + ClusterType ClusterType + // ClusterName is the clusterName being watched for through CDS. + ClusterName string + // EDSServiceName is an optional name for EDS. If it's not set, the balancer + // should watch ClusterName for the EDS resources. + EDSServiceName string + // EnableLRS indicates whether or not load should be reported through LRS. + EnableLRS bool + // SecurityCfg contains security configuration sent by the control plane. + SecurityCfg *SecurityConfig + // MaxRequests for circuit breaking, if any (otherwise nil). + MaxRequests *uint32 + // DNSHostName is used only for cluster type DNS. It's the DNS name to + // resolve in "host:port" form + DNSHostName string + // PrioritizedClusterNames is used only for cluster type aggregate. It represents + // a prioritized list of cluster names. + PrioritizedClusterNames []string + + // LBPolicy is the lb policy for this cluster. + // + // This only support round_robin and ring_hash. + // - if it's nil, the lb policy is round_robin + // - if it's not nil, the lb policy is ring_hash, the this field has the config. + // + // When we add more support policies, this can be made an interface, and + // will be set to different types based on the policy type. + LBPolicy *ClusterLBPolicyRingHash + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// ClusterUpdateErrTuple is a tuple with the update and error. It contains the +// results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type ClusterUpdateErrTuple struct { + Update ClusterUpdate + Err error +} diff --git a/xds/client/resource/type_eds.go b/xds/client/resource/type_eds.go new file mode 100644 index 0000000000..30627a6805 --- /dev/null +++ b/xds/client/resource/type_eds.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "google.golang.org/protobuf/types/known/anypb" +) + +// OverloadDropConfig contains the config to drop overloads. +type OverloadDropConfig struct { + Category string + Numerator uint32 + Denominator uint32 +} + +// EndpointHealthStatus represents the health status of an endpoint. +type EndpointHealthStatus int32 + +const ( + // EndpointHealthStatusUnknown represents HealthStatus UNKNOWN. + EndpointHealthStatusUnknown EndpointHealthStatus = iota + // EndpointHealthStatusHealthy represents HealthStatus HEALTHY. + EndpointHealthStatusHealthy + // EndpointHealthStatusUnhealthy represents HealthStatus UNHEALTHY. + EndpointHealthStatusUnhealthy + // EndpointHealthStatusDraining represents HealthStatus DRAINING. + EndpointHealthStatusDraining + // EndpointHealthStatusTimeout represents HealthStatus TIMEOUT. + EndpointHealthStatusTimeout + // EndpointHealthStatusDegraded represents HealthStatus DEGRADED. + EndpointHealthStatusDegraded +) + +// Endpoint contains information of an endpoint. +type Endpoint struct { + Address string + HealthStatus EndpointHealthStatus + Weight uint32 +} + +// Locality contains information of a locality. +type Locality struct { + Endpoints []Endpoint + ID LocalityID + Priority uint32 + Weight uint32 +} + +// EndpointsUpdate contains an EDS update. +type EndpointsUpdate struct { + Drops []OverloadDropConfig + Localities []Locality + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// EndpointsUpdateErrTuple is a tuple with the update and error. It contains the +// results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type EndpointsUpdateErrTuple struct { + Update EndpointsUpdate + Err error +} diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go new file mode 100644 index 0000000000..fb5f3af56c --- /dev/null +++ b/xds/client/resource/type_lds.go @@ -0,0 +1,87 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "google.golang.org/protobuf/types/known/anypb" +) + +// ListenerUpdate contains information received in an LDS response, which is of +// interest to the registered LDS watcher. +type ListenerUpdate struct { + // RouteConfigName is the route configuration name corresponding to the + // target which is being watched through LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + RouteConfigName string + // InlineRouteConfig is the inline route configuration (RDS response) + // returned inside LDS. + // + // Exactly one of RouteConfigName and InlineRouteConfig is set. + InlineRouteConfig *RouteConfigUpdate + + // MaxStreamDuration contains the HTTP connection manager's + // common_http_protocol_options.max_stream_duration field, or zero if + // unset. + MaxStreamDuration time.Duration + // HTTPFilters is a list of HTTP filters (name, config) from the LDS + // response. + HTTPFilters []HTTPFilter + // InboundListenerCfg contains inbound listener configuration. + InboundListenerCfg *InboundListenerConfig + + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// HTTPFilter represents one HTTP filter from an LDS response's HTTP connection +// manager field. +type HTTPFilter struct { + // Name is an arbitrary name of the filter. Used for applying override + // settings in virtual host / route / weighted cluster configuration (not + // yet supported). + Name string + // Filter is the HTTP filter found in the registry for the config type. + Filter httpfilter.Filter + // Config contains the filter's configuration + Config httpfilter.FilterConfig +} + +// InboundListenerConfig contains information about the inbound listener, i.e +// the server-side listener. +type InboundListenerConfig struct { + // Address is the local address on which the inbound listener is expected to + // accept incoming connections. + Address string + // Port is the local port on which the inbound listener is expected to + // accept incoming connections. + Port string + // FilterChains is the list of filter chains associated with this listener. + FilterChains *FilterChainManager +} + +// ListenerUpdateErrTuple is a tuple with the update and error. It contains the +// results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type ListenerUpdateErrTuple struct { + Update ListenerUpdate + Err error +} diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go new file mode 100644 index 0000000000..75efd46fcb --- /dev/null +++ b/xds/client/resource/type_rds.go @@ -0,0 +1,255 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "regexp" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) + +// RouteConfigUpdate contains information received in an RDS response, which is +// of interest to the registered RDS watcher. +type RouteConfigUpdate struct { + VirtualHosts []*VirtualHost + // ClusterSpecifierPlugins are the LB Configurations for any + // ClusterSpecifierPlugins referenced by the Route Table. + ClusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig + // Raw is the resource from the xds response. + Raw *anypb.Any +} + +// VirtualHost contains the routes for a list of Domains. +// +// Note that the domains in this slice can be a wildcard, not an exact string. +// The consumer of this struct needs to find the best match for its hostname. +type VirtualHost struct { + Domains []string + // Routes contains a list of routes, each containing matchers and + // corresponding action. + Routes []*Route + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the virtual host which may be present. An individual filter's override + // may be unused if the matching Route contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig +} + +// RetryConfig contains all retry-related configuration in either a VirtualHost +// or Route. +type RetryConfig struct { + // RetryOn is a set of status codes on which to retry. Only Canceled, + // DeadlineExceeded, Internal, ResourceExhausted, and Unavailable are + // supported; any other values will be omitted. + RetryOn map[codes.Code]bool + NumRetries uint32 // maximum number of retry attempts + RetryBackoff RetryBackoff // retry backoff policy +} + +// RetryBackoff describes the backoff policy for retries. +type RetryBackoff struct { + BaseInterval time.Duration // initial backoff duration between attempts + MaxInterval time.Duration // maximum backoff duration +} + +// HashPolicyType specifies the type of HashPolicy from a received RDS Response. +type HashPolicyType int + +const ( + // HashPolicyTypeHeader specifies to hash a Header in the incoming request. + HashPolicyTypeHeader HashPolicyType = iota + // HashPolicyTypeChannelID specifies to hash a unique Identifier of the + // Channel. In grpc-go, this will be done using the ClientConn pointer. + HashPolicyTypeChannelID +) + +// HashPolicy specifies the HashPolicy if the upstream cluster uses a hashing +// load balancer. +type HashPolicy struct { + HashPolicyType HashPolicyType + Terminal bool + // Fields used for type HEADER. + HeaderName string + Regex *regexp.Regexp + RegexSubstitution string +} + +// RouteActionType is the action of the route from a received RDS response. +type RouteActionType int + +const ( + // RouteActionUnsupported are routing types currently unsupported by grpc. + // According to A36, "A Route with an inappropriate action causes RPCs + // matching that route to fail." + RouteActionUnsupported RouteActionType = iota + // RouteActionRoute is the expected route type on the client side. Route + // represents routing a request to some upstream cluster. On the client + // side, if an RPC matches to a route that is not RouteActionRoute, the RPC + // will fail according to A36. + RouteActionRoute + // RouteActionNonForwardingAction is the expected route type on the server + // side. NonForwardingAction represents when a route will generate a + // response directly, without forwarding to an upstream host. + RouteActionNonForwardingAction +) + +// Route is both a specification of how to match a request as well as an +// indication of the action to take upon match. +type Route struct { + Path *string + Prefix *string + Regex *regexp.Regexp + // Indicates if prefix/path matching should be case insensitive. The default + // is false (case sensitive). + CaseInsensitive bool + Headers []*HeaderMatcher + Fraction *uint32 + + HashPolicies []*HashPolicy + + // If the matchers above indicate a match, the below configuration is used. + // If MaxStreamDuration is nil, it indicates neither of the route action's + // max_stream_duration fields (grpc_timeout_header_max nor + // max_stream_duration) were set. In this case, the ListenerUpdate's + // MaxStreamDuration field should be used. If MaxStreamDuration is set to + // an explicit zero duration, the application's deadline should be used. + MaxStreamDuration *time.Duration + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the route which may be present. An individual filter's override may be + // unused if the matching WeightedCluster contains an override for that + // filter. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig + RetryConfig *RetryConfig + + ActionType RouteActionType + + // Only one of the following fields (WeightedClusters or + // ClusterSpecifierPlugin) will be set for a route. + WeightedClusters map[string]WeightedCluster + // ClusterSpecifierPlugin is the name of the Cluster Specifier Plugin that + // this Route is linked to, if specified by xDS. + ClusterSpecifierPlugin string +} + +// WeightedCluster contains settings for an xds ActionType.WeightedCluster. +type WeightedCluster struct { + // Weight is the relative weight of the cluster. It will never be zero. + Weight uint32 + // HTTPFilterConfigOverride contains any HTTP filter config overrides for + // the weighted cluster which may be present. + HTTPFilterConfigOverride map[string]httpfilter.FilterConfig +} + +// HeaderMatcher represents header matchers. +type HeaderMatcher struct { + Name string + InvertMatch *bool + ExactMatch *string + RegexMatch *regexp.Regexp + PrefixMatch *string + SuffixMatch *string + RangeMatch *Int64Range + PresentMatch *bool +} + +// Int64Range is a range for header range match. +type Int64Range struct { + Start int64 + End int64 +} + +// SecurityConfig contains the security configuration received as part of the +// Cluster resource on the client-side, and as part of the Listener resource on +// the server-side. +type SecurityConfig struct { + // RootInstanceName identifies the certProvider plugin to be used to fetch + // root certificates. This instance name will be resolved to the plugin name + // and its associated configuration from the certificate_providers field of + // the bootstrap file. + RootInstanceName string + // RootCertName is the certificate name to be passed to the plugin (looked + // up from the bootstrap file) while fetching root certificates. + RootCertName string + // IdentityInstanceName identifies the certProvider plugin to be used to + // fetch identity certificates. This instance name will be resolved to the + // plugin name and its associated configuration from the + // certificate_providers field of the bootstrap file. + IdentityInstanceName string + // IdentityCertName is the certificate name to be passed to the plugin + // (looked up from the bootstrap file) while fetching identity certificates. + IdentityCertName string + // SubjectAltNameMatchers is an optional list of match criteria for SANs + // specified on the peer certificate. Used only on the client-side. + // + // Some intricacies: + // - If this field is empty, then any peer certificate is accepted. + // - If the peer certificate contains a wildcard DNS SAN, and an `exact` + // matcher is configured, a wildcard DNS match is performed instead of a + // regular string comparison. + SubjectAltNameMatchers []matcher.StringMatcher + // RequireClientCert indicates if the server handshake process expects the + // client to present a certificate. Set to true when performing mTLS. Used + // only on the server-side. + RequireClientCert bool +} + +// Equal returns true if sc is equal to other. +func (sc *SecurityConfig) Equal(other *SecurityConfig) bool { + switch { + case sc == nil && other == nil: + return true + case (sc != nil) != (other != nil): + return false + } + switch { + case sc.RootInstanceName != other.RootInstanceName: + return false + case sc.RootCertName != other.RootCertName: + return false + case sc.IdentityInstanceName != other.IdentityInstanceName: + return false + case sc.IdentityCertName != other.IdentityCertName: + return false + case sc.RequireClientCert != other.RequireClientCert: + return false + default: + if len(sc.SubjectAltNameMatchers) != len(other.SubjectAltNameMatchers) { + return false + } + for i := 0; i < len(sc.SubjectAltNameMatchers); i++ { + if !sc.SubjectAltNameMatchers[i].Equal(other.SubjectAltNameMatchers[i]) { + return false + } + } + } + return true +} + +// RouteConfigUpdateErrTuple is a tuple with the update and error. It contains +// the results from unmarshal functions. It's used to pass unmarshal results of +// multiple resources together, e.g. in maps like `map[string]{Update,error}`. +type RouteConfigUpdateErrTuple struct { + Update RouteConfigUpdate + Err error +} diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go new file mode 100644 index 0000000000..c19f6e2ff1 --- /dev/null +++ b/xds/client/resource/unmarshal.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package resource contains functions to proto xds updates (unmarshal from +// proto), and types for the resource updates. +package resource + +import ( + "errors" + "fmt" + "strings" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/protobuf/types/known/anypb" +) + +// UnmarshalOptions wraps the input parameters for `UnmarshalXxx` functions. +type UnmarshalOptions struct { + // Version is the version of the received response. + Version string + // Resources are the xDS resources resources in the received response. + Resources []*anypb.Any + // Logger is the prefix logger to be used during unmarshaling. + Logger *grpclog.PrefixLogger + // UpdateValidator is a post unmarshal validation check provided by the + // upper layer. + UpdateValidator UpdateValidatorFunc +} + +// processAllResources unmarshals and validates the resources, populates the +// provided ret (a map), and returns metadata and error. +// +// After this function, the ret map will be populated with both valid and +// invalid updates. Invalid resources will have an entry with the key as the +// resource name, value as an empty update. +// +// The type of the resource is determined by the type of ret. E.g. +// map[string]ListenerUpdate means this is for LDS. +func processAllResources(opts *UnmarshalOptions, ret interface{}) (UpdateMetadata, error) { + timestamp := time.Now() + md := UpdateMetadata{ + Version: opts.Version, + Timestamp: timestamp, + } + var topLevelErrors []error + perResourceErrors := make(map[string]error) + + for _, r := range opts.Resources { + switch ret2 := ret.(type) { + case map[string]ListenerUpdateErrTuple: + name, update, err := unmarshalListenerResource(r, opts.UpdateValidator, opts.Logger) + name = ParseName(name).String() + if err == nil { + ret2[name] = ListenerUpdateErrTuple{Update: update} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret2[name] = ListenerUpdateErrTuple{Err: err} + case map[string]RouteConfigUpdateErrTuple: + name, update, err := unmarshalRouteConfigResource(r, opts.Logger) + name = ParseName(name).String() + if err == nil { + ret2[name] = RouteConfigUpdateErrTuple{Update: update} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret2[name] = RouteConfigUpdateErrTuple{Err: err} + case map[string]ClusterUpdateErrTuple: + name, update, err := unmarshalClusterResource(r, opts.UpdateValidator, opts.Logger) + name = ParseName(name).String() + if err == nil { + ret2[name] = ClusterUpdateErrTuple{Update: update} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret2[name] = ClusterUpdateErrTuple{Err: err} + case map[string]EndpointsUpdateErrTuple: + name, update, err := unmarshalEndpointsResource(r, opts.Logger) + name = ParseName(name).String() + if err == nil { + ret2[name] = EndpointsUpdateErrTuple{Update: update} + continue + } + if name == "" { + topLevelErrors = append(topLevelErrors, err) + continue + } + perResourceErrors[name] = err + // Add place holder in the map so we know this resource name was in + // the response. + ret2[name] = EndpointsUpdateErrTuple{Err: err} + } + } + + if len(topLevelErrors) == 0 && len(perResourceErrors) == 0 { + md.Status = ServiceStatusACKed + return md, nil + } + + var typeStr string + switch ret.(type) { + case map[string]ListenerUpdate: + typeStr = "LDS" + case map[string]RouteConfigUpdate: + typeStr = "RDS" + case map[string]ClusterUpdate: + typeStr = "CDS" + case map[string]EndpointsUpdate: + typeStr = "EDS" + } + + md.Status = ServiceStatusNACKed + errRet := combineErrors(typeStr, topLevelErrors, perResourceErrors) + md.ErrState = &UpdateErrorMetadata{ + Version: opts.Version, + Err: errRet, + Timestamp: timestamp, + } + return md, errRet +} + +func combineErrors(rType string, topLevelErrors []error, perResourceErrors map[string]error) error { + var errStrB strings.Builder + errStrB.WriteString(fmt.Sprintf("error parsing %q response: ", rType)) + if len(topLevelErrors) > 0 { + errStrB.WriteString("top level errors: ") + for i, err := range topLevelErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + errStrB.WriteString(err.Error()) + } + } + if len(perResourceErrors) > 0 { + var i int + for name, err := range perResourceErrors { + if i != 0 { + errStrB.WriteString(";\n") + } + i++ + errStrB.WriteString(fmt.Sprintf("resource %q: %v", name, err.Error())) + } + } + return errors.New(errStrB.String()) +} diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go new file mode 100644 index 0000000000..e1dfc7934d --- /dev/null +++ b/xds/client/resource/unmarshal_cds.go @@ -0,0 +1,456 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "errors" + "fmt" + "net" + "strconv" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// TransportSocket proto message has a `name` field which is expected to be set +// to this value by the management server. +const transportSocketName = "envoy.transport_sockets.tls" + +// UnmarshalCluster processes resources received in an CDS response, validates +// them, and transforms them into a native struct which contains only fields we +// are interested in. +func UnmarshalCluster(opts *UnmarshalOptions) (map[string]ClusterUpdateErrTuple, UpdateMetadata, error) { + update := make(map[string]ClusterUpdateErrTuple) + md, err := processAllResources(opts, update) + return update, md, err +} + +func unmarshalClusterResource(r *anypb.Any, f UpdateValidatorFunc, logger *grpclog.PrefixLogger) (string, ClusterUpdate, error) { + if !IsClusterResource(r.GetTypeUrl()) { + return "", ClusterUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cluster := &v3clusterpb.Cluster{} + if err := proto.Unmarshal(r.GetValue(), cluster); err != nil { + return "", ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + logger.Infof("Resource with name: %v, type: %T, contains: %v", cluster.GetName(), cluster, pretty.ToJSON(cluster)) + cu, err := validateClusterAndConstructClusterUpdate(cluster) + if err != nil { + return cluster.GetName(), ClusterUpdate{}, err + } + cu.Raw = r + if f != nil { + if err := f(cu); err != nil { + return "", ClusterUpdate{}, err + } + } + + return cluster.GetName(), cu, nil +} + +const ( + defaultRingHashMinSize = 1024 + defaultRingHashMaxSize = 8 * 1024 * 1024 // 8M + ringHashSizeUpperBound = 8 * 1024 * 1024 // 8M +) + +func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { + var lbPolicy *ClusterLBPolicyRingHash + switch cluster.GetLbPolicy() { + case v3clusterpb.Cluster_ROUND_ROBIN: + lbPolicy = nil // The default is round_robin, and there's no config to set. + case v3clusterpb.Cluster_RING_HASH: + if !envconfig.XDSRingHash { + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + rhc := cluster.GetRingHashLbConfig() + if rhc.GetHashFunction() != v3clusterpb.Cluster_RingHashLbConfig_XX_HASH { + return ClusterUpdate{}, fmt.Errorf("unsupported ring_hash hash function %v in response: %+v", rhc.GetHashFunction(), cluster) + } + // Minimum defaults to 1024 entries, and limited to 8M entries Maximum + // defaults to 8M entries, and limited to 8M entries + var minSize, maxSize uint64 = defaultRingHashMinSize, defaultRingHashMaxSize + if min := rhc.GetMinimumRingSize(); min != nil { + if min.GetValue() > ringHashSizeUpperBound { + return ClusterUpdate{}, fmt.Errorf("unexpected ring_hash mininum ring size %v in response: %+v", min.GetValue(), cluster) + } + minSize = min.GetValue() + } + if max := rhc.GetMaximumRingSize(); max != nil { + if max.GetValue() > ringHashSizeUpperBound { + return ClusterUpdate{}, fmt.Errorf("unexpected ring_hash maxinum ring size %v in response: %+v", max.GetValue(), cluster) + } + maxSize = max.GetValue() + } + if minSize > maxSize { + return ClusterUpdate{}, fmt.Errorf("ring_hash config min size %v is greater than max %v", minSize, maxSize) + } + lbPolicy = &ClusterLBPolicyRingHash{MinimumRingSize: minSize, MaximumRingSize: maxSize} + default: + return ClusterUpdate{}, fmt.Errorf("unexpected lbPolicy %v in response: %+v", cluster.GetLbPolicy(), cluster) + } + + // Process security configuration received from the control plane iff the + // corresponding environment variable is set. + var sc *SecurityConfig + if envconfig.XDSClientSideSecurity { + var err error + if sc, err = securityConfigFromCluster(cluster); err != nil { + return ClusterUpdate{}, err + } + } + + ret := ClusterUpdate{ + ClusterName: cluster.GetName(), + EnableLRS: cluster.GetLrsServer().GetSelf() != nil, + SecurityCfg: sc, + MaxRequests: circuitBreakersFromCluster(cluster), + LBPolicy: lbPolicy, + } + + // Validate and set cluster type from the response. + switch { + case cluster.GetType() == v3clusterpb.Cluster_EDS: + if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil { + return ClusterUpdate{}, fmt.Errorf("unexpected edsConfig in response: %+v", cluster) + } + ret.ClusterType = ClusterTypeEDS + ret.EDSServiceName = cluster.GetEdsClusterConfig().GetServiceName() + return ret, nil + case cluster.GetType() == v3clusterpb.Cluster_LOGICAL_DNS: + if !envconfig.XDSAggregateAndDNS { + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } + ret.ClusterType = ClusterTypeLogicalDNS + dnsHN, err := dnsHostNameFromCluster(cluster) + if err != nil { + return ClusterUpdate{}, err + } + ret.DNSHostName = dnsHN + return ret, nil + case cluster.GetClusterType() != nil && cluster.GetClusterType().Name == "envoy.clusters.aggregate": + if !envconfig.XDSAggregateAndDNS { + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } + clusters := &v3aggregateclusterpb.ClusterConfig{} + if err := proto.Unmarshal(cluster.GetClusterType().GetTypedConfig().GetValue(), clusters); err != nil { + return ClusterUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + ret.ClusterType = ClusterTypeAggregate + ret.PrioritizedClusterNames = clusters.Clusters + return ret, nil + default: + return ClusterUpdate{}, fmt.Errorf("unsupported cluster type (%v, %v) in response: %+v", cluster.GetType(), cluster.GetClusterType(), cluster) + } +} + +// dnsHostNameFromCluster extracts the DNS host name from the cluster's load +// assignment. +// +// There should be exactly one locality, with one endpoint, whose address +// contains the address and port. +func dnsHostNameFromCluster(cluster *v3clusterpb.Cluster) (string, error) { + loadAssignment := cluster.GetLoadAssignment() + if loadAssignment == nil { + return "", fmt.Errorf("load_assignment not present for LOGICAL_DNS cluster") + } + if len(loadAssignment.GetEndpoints()) != 1 { + return "", fmt.Errorf("load_assignment for LOGICAL_DNS cluster must have exactly one locality, got: %+v", loadAssignment) + } + endpoints := loadAssignment.GetEndpoints()[0].GetLbEndpoints() + if len(endpoints) != 1 { + return "", fmt.Errorf("locality for LOGICAL_DNS cluster must have exactly one endpoint, got: %+v", endpoints) + } + endpoint := endpoints[0].GetEndpoint() + if endpoint == nil { + return "", fmt.Errorf("endpoint for LOGICAL_DNS cluster not set") + } + socketAddr := endpoint.GetAddress().GetSocketAddress() + if socketAddr == nil { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set") + } + if socketAddr.GetResolverName() != "" { + return "", fmt.Errorf("socket address for endpoint for LOGICAL_DNS cluster not set has unexpected custom resolver name: %v", socketAddr.GetResolverName()) + } + host := socketAddr.GetAddress() + if host == "" { + return "", fmt.Errorf("host for endpoint for LOGICAL_DNS cluster not set") + } + port := socketAddr.GetPortValue() + if port == 0 { + return "", fmt.Errorf("port for endpoint for LOGICAL_DNS cluster not set") + } + return net.JoinHostPort(host, strconv.Itoa(int(port))), nil +} + +// securityConfigFromCluster extracts the relevant security configuration from +// the received Cluster resource. +func securityConfigFromCluster(cluster *v3clusterpb.Cluster) (*SecurityConfig, error) { + if tsm := cluster.GetTransportSocketMatches(); len(tsm) != 0 { + return nil, fmt.Errorf("unsupport transport_socket_matches field is non-empty: %+v", tsm) + } + // The Cluster resource contains a `transport_socket` field, which contains + // a oneof `typed_config` field of type `protobuf.Any`. The any proto + // contains a marshaled representation of an `UpstreamTlsContext` message. + ts := cluster.GetTransportSocket() + if ts == nil { + return nil, nil + } + if name := ts.GetName(); name != transportSocketName { + return nil, fmt.Errorf("transport_socket field has unexpected name: %s", name) + } + any := ts.GetTypedConfig() + if any == nil || any.TypeUrl != version.V3UpstreamTLSContextURL { + return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl) + } + upstreamCtx := &v3tlspb.UpstreamTlsContext{} + if err := proto.Unmarshal(any.GetValue(), upstreamCtx); err != nil { + return nil, fmt.Errorf("failed to unmarshal UpstreamTlsContext in CDS response: %v", err) + } + // The following fields from `UpstreamTlsContext` are ignored: + // - sni + // - allow_renegotiation + // - max_session_keys + if upstreamCtx.GetCommonTlsContext() == nil { + return nil, errors.New("UpstreamTlsContext in CDS response does not contain a CommonTlsContext") + } + + return securityConfigFromCommonTLSContext(upstreamCtx.GetCommonTlsContext(), false) +} + +// common is expected to be not nil. +// The `alpn_protocols` field is ignored. +func securityConfigFromCommonTLSContext(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + if common.GetTlsParams() != nil { + return nil, fmt.Errorf("unsupported tls_params field in CommonTlsContext message: %+v", common) + } + if common.GetCustomHandshaker() != nil { + return nil, fmt.Errorf("unsupported custom_handshaker field in CommonTlsContext message: %+v", common) + } + + // For now, if we can't get a valid security config from the new fields, we + // fallback to the old deprecated fields. + // TODO: Drop support for deprecated fields. NACK if err != nil here. + sc, _ := securityConfigFromCommonTLSContextUsingNewFields(common, server) + if sc == nil || sc.Equal(&SecurityConfig{}) { + var err error + sc, err = securityConfigFromCommonTLSContextWithDeprecatedFields(common, server) + if err != nil { + return nil, err + } + } + if sc != nil { + // sc == nil is a valid case where the control plane has not sent us any + // security configuration. xDS creds will use fallback creds. + if server { + if sc.IdentityInstanceName == "" { + return nil, errors.New("security configuration on the server-side does not contain identity certificate provider instance name") + } + } else { + if sc.RootInstanceName == "" { + return nil, errors.New("security configuration on the client-side does not contain root certificate provider instance name") + } + } + } + return sc, nil +} + +func securityConfigFromCommonTLSContextWithDeprecatedFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `CommonTlsContext` contains a + // `tls_certificate_certificate_provider_instance` field of type + // `CertificateProviderInstance`, which contains the provider instance name + // and the certificate name to fetch identity certs. + sc := &SecurityConfig{} + if identity := common.GetTlsCertificateCertificateProviderInstance(); identity != nil { + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + } + + // The `CommonTlsContext` contains a `validation_context_type` field which + // is a oneof. We can get the values that we are interested in from two of + // those possible values: + // - combined validation context: + // - contains a default validation context which holds the list of + // matchers for accepted SANs. + // - contains certificate provider instance configuration + // - certificate provider instance configuration + // - in this case, we do not get a list of accepted SANs. + switch t := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + combined := common.GetCombinedValidationContext() + var matchers []matcher.StringMatcher + if def := combined.GetDefaultValidationContext(); def != nil { + for _, m := range def.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + if pi := combined.GetValidationContextCertificateProviderInstance(); pi != nil { + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + } + case *v3tlspb.CommonTlsContext_ValidationContextCertificateProviderInstance: + pi := common.GetValidationContextCertificateProviderInstance() + sc.RootInstanceName = pi.GetInstanceName() + sc.RootCertName = pi.GetCertificateName() + case nil: + // It is valid for the validation context to be nil on the server side. + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", t) + } + return sc, nil +} + +// gRFC A29 https://github.com/grpc/proposal/blob/master/A29-xds-tls-security.md +// specifies the new way to fetch security configuration and says the following: +// +// Although there are various ways to obtain certificates as per this proto +// (which are supported by Envoy), gRPC supports only one of them and that is +// the `CertificateProviderPluginInstance` proto. +// +// This helper function attempts to fetch security configuration from the +// `CertificateProviderPluginInstance` message, given a CommonTlsContext. +func securityConfigFromCommonTLSContextUsingNewFields(common *v3tlspb.CommonTlsContext, server bool) (*SecurityConfig, error) { + // The `tls_certificate_provider_instance` field of type + // `CertificateProviderPluginInstance` is used to fetch the identity + // certificate provider. + sc := &SecurityConfig{} + identity := common.GetTlsCertificateProviderInstance() + if identity == nil && len(common.GetTlsCertificates()) != 0 { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificates is set in CommonTlsContext message: %+v", common) + } + if identity == nil && common.GetTlsCertificateSdsSecretConfigs() != nil { + return nil, fmt.Errorf("expected field tls_certificate_provider_instance is not set, while unsupported field tls_certificate_sds_secret_configs is set in CommonTlsContext message: %+v", common) + } + sc.IdentityInstanceName = identity.GetInstanceName() + sc.IdentityCertName = identity.GetCertificateName() + + // The `CommonTlsContext` contains a oneof field `validation_context_type`, + // which contains the `CertificateValidationContext` message in one of the + // following ways: + // - `validation_context` field + // - this is directly of type `CertificateValidationContext` + // - `combined_validation_context` field + // - this is of type `CombinedCertificateValidationContext` and contains + // a `default validation context` field of type + // `CertificateValidationContext` + // + // The `CertificateValidationContext` message has the following fields that + // we are interested in: + // - `ca_certificate_provider_instance` + // - this is of type `CertificateProviderPluginInstance` + // - `match_subject_alt_names` + // - this is a list of string matchers + // + // The `CertificateProviderPluginInstance` message contains two fields + // - instance_name + // - this is the certificate provider instance name to be looked up in + // the bootstrap configuration + // - certificate_name + // - this is an opaque name passed to the certificate provider + var validationCtx *v3tlspb.CertificateValidationContext + switch typ := common.GetValidationContextType().(type) { + case *v3tlspb.CommonTlsContext_ValidationContext: + validationCtx = common.GetValidationContext() + case *v3tlspb.CommonTlsContext_CombinedValidationContext: + validationCtx = common.GetCombinedValidationContext().GetDefaultValidationContext() + case nil: + // It is valid for the validation context to be nil on the server side. + return sc, nil + default: + return nil, fmt.Errorf("validation context contains unexpected type: %T", typ) + } + // If we get here, it means that the `CertificateValidationContext` message + // was found through one of the supported ways. It is an error if the + // validation context is specified, but it does not contain the + // ca_certificate_provider_instance field which contains information about + // the certificate provider to be used for the root certificates. + if validationCtx.GetCaCertificateProviderInstance() == nil { + return nil, fmt.Errorf("expected field ca_certificate_provider_instance is missing in CommonTlsContext message: %+v", common) + } + // The following fields are ignored: + // - trusted_ca + // - watched_directory + // - allow_expired_certificate + // - trust_chain_verification + switch { + case len(validationCtx.GetVerifyCertificateSpki()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_spki field in CommonTlsContext message: %+v", common) + case len(validationCtx.GetVerifyCertificateHash()) != 0: + return nil, fmt.Errorf("unsupported verify_certificate_hash field in CommonTlsContext message: %+v", common) + case validationCtx.GetRequireSignedCertificateTimestamp().GetValue(): + return nil, fmt.Errorf("unsupported require_sugned_ceritificate_timestamp field in CommonTlsContext message: %+v", common) + case validationCtx.GetCrl() != nil: + return nil, fmt.Errorf("unsupported crl field in CommonTlsContext message: %+v", common) + case validationCtx.GetCustomValidatorConfig() != nil: + return nil, fmt.Errorf("unsupported custom_validator_config field in CommonTlsContext message: %+v", common) + } + + if rootProvider := validationCtx.GetCaCertificateProviderInstance(); rootProvider != nil { + sc.RootInstanceName = rootProvider.GetInstanceName() + sc.RootCertName = rootProvider.GetCertificateName() + } + var matchers []matcher.StringMatcher + for _, m := range validationCtx.GetMatchSubjectAltNames() { + matcher, err := matcher.StringMatcherFromProto(m) + if err != nil { + return nil, err + } + matchers = append(matchers, matcher) + } + if server && len(matchers) != 0 { + return nil, fmt.Errorf("match_subject_alt_names field in validation context is not supported on the server: %v", common) + } + sc.SubjectAltNameMatchers = matchers + return sc, nil +} + +// circuitBreakersFromCluster extracts the circuit breakers configuration from +// the received cluster resource. Returns nil if no CircuitBreakers or no +// Thresholds in CircuitBreakers. +func circuitBreakersFromCluster(cluster *v3clusterpb.Cluster) *uint32 { + for _, threshold := range cluster.GetCircuitBreakers().GetThresholds() { + if threshold.GetPriority() != v3corepb.RoutingPriority_DEFAULT { + continue + } + maxRequestsPb := threshold.GetMaxRequests() + if maxRequestsPb == nil { + return nil + } + maxRequests := maxRequestsPb.GetValue() + return &maxRequests + } + return nil +} diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go new file mode 100644 index 0000000000..f4403ca371 --- /dev/null +++ b/xds/client/resource/unmarshal_eds.go @@ -0,0 +1,130 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "fmt" + "net" + "strconv" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +// UnmarshalEndpoints processes resources received in an EDS response, +// validates them, and transforms them into a native struct which contains only +// fields we are interested in. +func UnmarshalEndpoints(opts *UnmarshalOptions) (map[string]EndpointsUpdateErrTuple, UpdateMetadata, error) { + update := make(map[string]EndpointsUpdateErrTuple) + md, err := processAllResources(opts, update) + return update, md, err +} + +func unmarshalEndpointsResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, EndpointsUpdate, error) { + if !IsEndpointsResource(r.GetTypeUrl()) { + return "", EndpointsUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + + cla := &v3endpointpb.ClusterLoadAssignment{} + if err := proto.Unmarshal(r.GetValue(), cla); err != nil { + return "", EndpointsUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + logger.Infof("Resource with name: %v, type: %T, contains: %v", cla.GetClusterName(), cla, pretty.ToJSON(cla)) + + u, err := parseEDSRespProto(cla) + if err != nil { + return cla.GetClusterName(), EndpointsUpdate{}, err + } + u.Raw = r + return cla.GetClusterName(), u, nil +} + +func parseAddress(socketAddress *v3corepb.SocketAddress) string { + return net.JoinHostPort(socketAddress.GetAddress(), strconv.Itoa(int(socketAddress.GetPortValue()))) +} + +func parseDropPolicy(dropPolicy *v3endpointpb.ClusterLoadAssignment_Policy_DropOverload) OverloadDropConfig { + percentage := dropPolicy.GetDropPercentage() + var ( + numerator = percentage.GetNumerator() + denominator uint32 + ) + switch percentage.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + denominator = 100 + case v3typepb.FractionalPercent_TEN_THOUSAND: + denominator = 10000 + case v3typepb.FractionalPercent_MILLION: + denominator = 1000000 + } + return OverloadDropConfig{ + Category: dropPolicy.GetCategory(), + Numerator: numerator, + Denominator: denominator, + } +} + +func parseEndpoints(lbEndpoints []*v3endpointpb.LbEndpoint) []Endpoint { + endpoints := make([]Endpoint, 0, len(lbEndpoints)) + for _, lbEndpoint := range lbEndpoints { + endpoints = append(endpoints, Endpoint{ + HealthStatus: EndpointHealthStatus(lbEndpoint.GetHealthStatus()), + Address: parseAddress(lbEndpoint.GetEndpoint().GetAddress().GetSocketAddress()), + Weight: lbEndpoint.GetLoadBalancingWeight().GetValue(), + }) + } + return endpoints +} + +func parseEDSRespProto(m *v3endpointpb.ClusterLoadAssignment) (EndpointsUpdate, error) { + ret := EndpointsUpdate{} + for _, dropPolicy := range m.GetPolicy().GetDropOverloads() { + ret.Drops = append(ret.Drops, parseDropPolicy(dropPolicy)) + } + priorities := make(map[uint32]struct{}) + for _, locality := range m.Endpoints { + l := locality.GetLocality() + if l == nil { + return EndpointsUpdate{}, fmt.Errorf("EDS response contains a locality without ID, locality: %+v", locality) + } + lid := LocalityID{ + Region: l.Region, + Zone: l.Zone, + SubZone: l.SubZone, + } + priority := locality.GetPriority() + priorities[priority] = struct{}{} + ret.Localities = append(ret.Localities, Locality{ + ID: lid, + Endpoints: parseEndpoints(locality.GetLbEndpoints()), + Weight: locality.GetLoadBalancingWeight().GetValue(), + Priority: priority, + }) + } + for i := 0; i < len(priorities); i++ { + if _, ok := priorities[uint32(i)]; !ok { + return EndpointsUpdate{}, fmt.Errorf("priority %v missing (with different priorities %v received)", i, priorities) + } + } + return ret, nil +} diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go new file mode 100644 index 0000000000..4e949e0f05 --- /dev/null +++ b/xds/client/resource/unmarshal_lds.go @@ -0,0 +1,297 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "errors" + "fmt" + "strconv" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + v1udpatypepb "github.com/cncf/udpa/go/udpa/type/v1" + v3cncftypepb "github.com/cncf/xds/go/xds/type/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) + +// UnmarshalListener processes resources received in an LDS response, validates +// them, and transforms them into a native struct which contains only fields we +// are interested in. +func UnmarshalListener(opts *UnmarshalOptions) (map[string]ListenerUpdateErrTuple, UpdateMetadata, error) { + update := make(map[string]ListenerUpdateErrTuple) + md, err := processAllResources(opts, update) + return update, md, err +} + +func unmarshalListenerResource(r *anypb.Any, f UpdateValidatorFunc, logger *grpclog.PrefixLogger) (string, ListenerUpdate, error) { + if !IsListenerResource(r.GetTypeUrl()) { + return "", ListenerUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + // TODO: Pass version.TransportAPI instead of relying upon the type URL + v2 := r.GetTypeUrl() == version.V2ListenerURL + lis := &v3listenerpb.Listener{} + if err := proto.Unmarshal(r.GetValue(), lis); err != nil { + return "", ListenerUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + logger.Infof("Resource with name: %v, type: %T, contains: %v", lis.GetName(), lis, pretty.ToJSON(lis)) + + lu, err := processListener(lis, logger, v2) + if err != nil { + return lis.GetName(), ListenerUpdate{}, err + } + if f != nil { + if err := f(*lu); err != nil { + return lis.GetName(), ListenerUpdate{}, err + } + } + lu.Raw = r + return lis.GetName(), *lu, nil +} + +func processListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger, v2 bool) (*ListenerUpdate, error) { + if lis.GetApiListener() != nil { + return processClientSideListener(lis, logger, v2) + } + return processServerSideListener(lis, logger) +} + +// processClientSideListener checks if the provided Listener proto meets +// the expected criteria. If so, it returns a non-empty routeConfigName. +func processClientSideListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger, v2 bool) (*ListenerUpdate, error) { + update := &ListenerUpdate{} + + apiLisAny := lis.GetApiListener().GetApiListener() + if !IsHTTPConnManagerResource(apiLisAny.GetTypeUrl()) { + return nil, fmt.Errorf("unexpected resource type: %q", apiLisAny.GetTypeUrl()) + } + apiLis := &v3httppb.HttpConnectionManager{} + if err := proto.Unmarshal(apiLisAny.GetValue(), apiLis); err != nil { + return nil, fmt.Errorf("failed to unmarshal api_listner: %v", err) + } + // "HttpConnectionManager.xff_num_trusted_hops must be unset or zero and + // HttpConnectionManager.original_ip_detection_extensions must be empty. If + // either field has an incorrect value, the Listener must be NACKed." - A41 + if apiLis.XffNumTrustedHops != 0 { + return nil, fmt.Errorf("xff_num_trusted_hops must be unset or zero %+v", apiLis) + } + if len(apiLis.OriginalIpDetectionExtensions) != 0 { + return nil, fmt.Errorf("original_ip_detection_extensions must be empty %+v", apiLis) + } + + switch apiLis.RouteSpecifier.(type) { + case *v3httppb.HttpConnectionManager_Rds: + if apiLis.GetRds().GetConfigSource().GetAds() == nil { + return nil, fmt.Errorf("ConfigSource is not ADS: %+v", lis) + } + name := apiLis.GetRds().GetRouteConfigName() + if name == "" { + return nil, fmt.Errorf("empty route_config_name: %+v", lis) + } + update.RouteConfigName = name + case *v3httppb.HttpConnectionManager_RouteConfig: + routeU, err := generateRDSUpdateFromRouteConfiguration(apiLis.GetRouteConfig(), logger, v2) + if err != nil { + return nil, fmt.Errorf("failed to parse inline RDS resp: %v", err) + } + update.InlineRouteConfig = &routeU + case nil: + return nil, fmt.Errorf("no RouteSpecifier: %+v", apiLis) + default: + return nil, fmt.Errorf("unsupported type %T for RouteSpecifier", apiLis.RouteSpecifier) + } + + if v2 { + return update, nil + } + + // The following checks and fields only apply to xDS protocol versions v3+. + + update.MaxStreamDuration = apiLis.GetCommonHttpProtocolOptions().GetMaxStreamDuration().AsDuration() + + var err error + if update.HTTPFilters, err = processHTTPFilters(apiLis.GetHttpFilters(), false); err != nil { + return nil, err + } + + return update, nil +} + +func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) { + switch { + case ptypes.Is(config, &v3cncftypepb.TypedStruct{}): + // The real type name is inside the new TypedStruct message. + s := new(v3cncftypepb.TypedStruct) + if err := ptypes.UnmarshalAny(config, s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + case ptypes.Is(config, &v1udpatypepb.TypedStruct{}): + // The real type name is inside the old TypedStruct message. + s := new(v1udpatypepb.TypedStruct) + if err := ptypes.UnmarshalAny(config, s); err != nil { + return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + } + return s, s.GetTypeUrl(), nil + default: + return config, config.GetTypeUrl(), nil + } +} + +func validateHTTPFilterConfig(cfg *anypb.Any, lds, optional bool) (httpfilter.Filter, httpfilter.FilterConfig, error) { + config, typeURL, err := unwrapHTTPFilterConfig(cfg) + if err != nil { + return nil, nil, err + } + filterBuilder := httpfilter.Get(typeURL) + if filterBuilder == nil { + if optional { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("no filter implementation found for %q", typeURL) + } + parseFunc := filterBuilder.ParseFilterConfig + if !lds { + parseFunc = filterBuilder.ParseFilterConfigOverride + } + filterConfig, err := parseFunc(config) + if err != nil { + return nil, nil, fmt.Errorf("error parsing config for filter %q: %v", typeURL, err) + } + return filterBuilder, filterConfig, nil +} + +func processHTTPFilterOverrides(cfgs map[string]*anypb.Any) (map[string]httpfilter.FilterConfig, error) { + if len(cfgs) == 0 { + return nil, nil + } + m := make(map[string]httpfilter.FilterConfig) + for name, cfg := range cfgs { + optional := false + s := new(v3routepb.FilterConfig) + if ptypes.Is(cfg, s) { + if err := ptypes.UnmarshalAny(cfg, s); err != nil { + return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err) + } + cfg = s.GetConfig() + optional = s.GetIsOptional() + } + + httpFilter, config, err := validateHTTPFilterConfig(cfg, false, optional) + if err != nil { + return nil, fmt.Errorf("filter override %q: %v", name, err) + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + m[name] = config + } + return m, nil +} + +func processHTTPFilters(filters []*v3httppb.HttpFilter, server bool) ([]HTTPFilter, error) { + ret := make([]HTTPFilter, 0, len(filters)) + seenNames := make(map[string]bool, len(filters)) + for _, filter := range filters { + name := filter.GetName() + if name == "" { + return nil, errors.New("filter missing name field") + } + if seenNames[name] { + return nil, fmt.Errorf("duplicate filter name %q", name) + } + seenNames[name] = true + + httpFilter, config, err := validateHTTPFilterConfig(filter.GetTypedConfig(), true, filter.GetIsOptional()) + if err != nil { + return nil, err + } + if httpFilter == nil { + // Optional configs are ignored. + continue + } + if server { + if _, ok := httpFilter.(httpfilter.ServerInterceptorBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported server-side", name) + } + } else if _, ok := httpFilter.(httpfilter.ClientInterceptorBuilder); !ok { + if filter.GetIsOptional() { + continue + } + return nil, fmt.Errorf("HTTP filter %q not supported client-side", name) + } + + // Save name/config + ret = append(ret, HTTPFilter{Name: name, Filter: httpFilter, Config: config}) + } + // "Validation will fail if a terminal filter is not the last filter in the + // chain or if a non-terminal filter is the last filter in the chain." - A39 + if len(ret) == 0 { + return nil, fmt.Errorf("http filters list is empty") + } + var i int + for ; i < len(ret)-1; i++ { + if ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is a terminal filter but it is not last in the filter chain", ret[i].Name) + } + } + if !ret[i].Filter.IsTerminal() { + return nil, fmt.Errorf("http filter %q is not a terminal filter", ret[len(ret)-1].Name) + } + return ret, nil +} + +func processServerSideListener(lis *v3listenerpb.Listener, logger *grpclog.PrefixLogger) (*ListenerUpdate, error) { + if n := len(lis.ListenerFilters); n != 0 { + return nil, fmt.Errorf("unsupported field 'listener_filters' contains %d entries", n) + } + if useOrigDst := lis.GetUseOriginalDst(); useOrigDst != nil && useOrigDst.GetValue() { + return nil, errors.New("unsupported field 'use_original_dst' is present and set to true") + } + addr := lis.GetAddress() + if addr == nil { + return nil, fmt.Errorf("no address field in LDS response: %+v", lis) + } + sockAddr := addr.GetSocketAddress() + if sockAddr == nil { + return nil, fmt.Errorf("no socket_address field in LDS response: %+v", lis) + } + lu := &ListenerUpdate{ + InboundListenerCfg: &InboundListenerConfig{ + Address: sockAddr.GetAddress(), + Port: strconv.Itoa(int(sockAddr.GetPortValue())), + }, + } + + fcMgr, err := NewFilterChainManager(lis, logger) + if err != nil { + return nil, err + } + lu.InboundListenerCfg.FilterChains = fcMgr + return lu, nil +} diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go new file mode 100644 index 0000000000..e8063a6bee --- /dev/null +++ b/xds/client/resource/unmarshal_rds.go @@ -0,0 +1,440 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package resource + +import ( + "fmt" + "regexp" + "strings" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) + +// UnmarshalRouteConfig processes resources received in an RDS response, +// validates them, and transforms them into a native struct which contains only +// fields we are interested in. The provided hostname determines the route +// configuration resources of interest. +func UnmarshalRouteConfig(opts *UnmarshalOptions) (map[string]RouteConfigUpdateErrTuple, UpdateMetadata, error) { + update := make(map[string]RouteConfigUpdateErrTuple) + md, err := processAllResources(opts, update) + return update, md, err +} + +func unmarshalRouteConfigResource(r *anypb.Any, logger *grpclog.PrefixLogger) (string, RouteConfigUpdate, error) { + if !IsRouteConfigResource(r.GetTypeUrl()) { + return "", RouteConfigUpdate{}, fmt.Errorf("unexpected resource type: %q ", r.GetTypeUrl()) + } + rc := &v3routepb.RouteConfiguration{} + if err := proto.Unmarshal(r.GetValue(), rc); err != nil { + return "", RouteConfigUpdate{}, fmt.Errorf("failed to unmarshal resource: %v", err) + } + logger.Infof("Resource with name: %v, type: %T, contains: %v.", rc.GetName(), rc, pretty.ToJSON(rc)) + + // TODO: Pass version.TransportAPI instead of relying upon the type URL + v2 := r.GetTypeUrl() == version.V2RouteConfigURL + u, err := generateRDSUpdateFromRouteConfiguration(rc, logger, v2) + if err != nil { + return rc.GetName(), RouteConfigUpdate{}, err + } + u.Raw = r + return rc.GetName(), u, nil +} + +// generateRDSUpdateFromRouteConfiguration checks if the provided +// RouteConfiguration meets the expected criteria. If so, it returns a +// RouteConfigUpdate with nil error. +// +// A RouteConfiguration resource is considered valid when only if it contains a +// VirtualHost whose domain field matches the server name from the URI passed +// to the gRPC channel, and it contains a clusterName or a weighted cluster. +// +// The RouteConfiguration includes a list of virtualHosts, which may have zero +// or more elements. We are interested in the element whose domains field +// matches the server name specified in the "xds:" URI. The only field in the +// VirtualHost proto that the we are interested in is the list of routes. We +// only look at the last route in the list (the default route), whose match +// field must be empty and whose route field must be set. Inside that route +// message, the cluster field will contain the clusterName or weighted clusters +// we are looking for. +func generateRDSUpdateFromRouteConfiguration(rc *v3routepb.RouteConfiguration, logger *grpclog.PrefixLogger, v2 bool) (RouteConfigUpdate, error) { + vhs := make([]*VirtualHost, 0, len(rc.GetVirtualHosts())) + csps := make(map[string]clusterspecifier.BalancerConfig) + if envconfig.XDSRLS { + var err error + csps, err = processClusterSpecifierPlugins(rc.ClusterSpecifierPlugins) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid %v", err) + } + } + // cspNames represents all the cluster specifiers referenced by Route + // Actions - any cluster specifiers not referenced by a Route Action can be + // ignored and not emitted by the xdsclient. + var cspNames = make(map[string]bool) + for _, vh := range rc.GetVirtualHosts() { + routes, cspNs, err := routesProtoToSlice(vh.Routes, csps, logger, v2) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + for n := range cspNs { + cspNames[n] = true + } + rc, err := generateRetryConfig(vh.GetRetryPolicy()) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("received route is invalid: %v", err) + } + vhOut := &VirtualHost{ + Domains: vh.GetDomains(), + Routes: routes, + RetryConfig: rc, + } + if !v2 { + cfgs, err := processHTTPFilterOverrides(vh.GetTypedPerFilterConfig()) + if err != nil { + return RouteConfigUpdate{}, fmt.Errorf("virtual host %+v: %v", vh, err) + } + vhOut.HTTPFilterConfigOverride = cfgs + } + vhs = append(vhs, vhOut) + } + + // "For any entry in the RouteConfiguration.cluster_specifier_plugins not + // referenced by an enclosed ActionType's cluster_specifier_plugin, the xDS + // client should not provide it to its consumers." - RLS in xDS Design + for name := range csps { + if !cspNames[name] { + delete(csps, name) + } + } + + return RouteConfigUpdate{VirtualHosts: vhs, ClusterSpecifierPlugins: csps}, nil +} + +func processClusterSpecifierPlugins(csps []*v3routepb.ClusterSpecifierPlugin) (map[string]clusterspecifier.BalancerConfig, error) { + cspCfgs := make(map[string]clusterspecifier.BalancerConfig) + // "The xDS client will inspect all elements of the + // cluster_specifier_plugins field looking up a plugin based on the + // extension.typed_config of each." - RLS in xDS design + for _, csp := range csps { + cs := clusterspecifier.Get(csp.GetExtension().GetTypedConfig().GetTypeUrl()) + if cs == nil { + // "If no plugin is registered for it, the resource will be NACKed." + // - RLS in xDS design + return nil, fmt.Errorf("cluster specifier %q of type %q was not found", csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + lbCfg, err := cs.ParseClusterSpecifierConfig(csp.GetExtension().GetTypedConfig()) + if err != nil { + // "If a plugin is found, the value of the typed_config field will + // be passed to it's conversion method, and if an error is + // encountered, the resource will be NACKED." - RLS in xDS design + return nil, fmt.Errorf("error: %q parsing config %q for cluster specifier %q of type %q", err, csp.GetExtension().GetTypedConfig(), csp.GetExtension().GetName(), csp.GetExtension().GetTypedConfig().GetTypeUrl()) + } + // "If all cluster specifiers are valid, the xDS client will store the + // configurations in a map keyed by the name of the extension instance." - + // RLS in xDS Design + cspCfgs[csp.GetExtension().GetName()] = lbCfg + } + return cspCfgs, nil +} + +func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) { + if rp == nil { + return nil, nil + } + + cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)} + for _, s := range strings.Split(rp.GetRetryOn(), ",") { + switch strings.TrimSpace(strings.ToLower(s)) { + case "cancelled": + cfg.RetryOn[codes.Canceled] = true + case "deadline-exceeded": + cfg.RetryOn[codes.DeadlineExceeded] = true + case "internal": + cfg.RetryOn[codes.Internal] = true + case "resource-exhausted": + cfg.RetryOn[codes.ResourceExhausted] = true + case "unavailable": + cfg.RetryOn[codes.Unavailable] = true + } + } + + if rp.NumRetries == nil { + cfg.NumRetries = 1 + } else { + cfg.NumRetries = rp.GetNumRetries().Value + if cfg.NumRetries < 1 { + return nil, fmt.Errorf("retry_policy.num_retries = %v; must be >= 1", cfg.NumRetries) + } + } + + backoff := rp.GetRetryBackOff() + if backoff == nil { + cfg.RetryBackoff.BaseInterval = 25 * time.Millisecond + } else { + cfg.RetryBackoff.BaseInterval = backoff.GetBaseInterval().AsDuration() + if cfg.RetryBackoff.BaseInterval <= 0 { + return nil, fmt.Errorf("retry_policy.base_interval = %v; must be > 0", cfg.RetryBackoff.BaseInterval) + } + } + if max := backoff.GetMaxInterval(); max == nil { + cfg.RetryBackoff.MaxInterval = 10 * cfg.RetryBackoff.BaseInterval + } else { + cfg.RetryBackoff.MaxInterval = max.AsDuration() + if cfg.RetryBackoff.MaxInterval <= 0 { + return nil, fmt.Errorf("retry_policy.max_interval = %v; must be > 0", cfg.RetryBackoff.MaxInterval) + } + } + + if len(cfg.RetryOn) == 0 { + return &RetryConfig{}, nil + } + return cfg, nil +} + +func routesProtoToSlice(routes []*v3routepb.Route, csps map[string]clusterspecifier.BalancerConfig, logger *grpclog.PrefixLogger, v2 bool) ([]*Route, map[string]bool, error) { + var routesRet []*Route + var cspNames = make(map[string]bool) + for _, r := range routes { + match := r.GetMatch() + if match == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a match", r) + } + + if len(match.GetQueryParameters()) != 0 { + // Ignore route with query parameters. + logger.Warningf("route %+v has query parameter matchers, the route will be ignored", r) + continue + } + + pathSp := match.GetPathSpecifier() + if pathSp == nil { + return nil, nil, fmt.Errorf("route %+v doesn't have a path specifier", r) + } + + var route Route + switch pt := pathSp.(type) { + case *v3routepb.RouteMatch_Prefix: + route.Prefix = &pt.Prefix + case *v3routepb.RouteMatch_Path: + route.Path = &pt.Path + case *v3routepb.RouteMatch_SafeRegex: + regex := pt.SafeRegex.GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + route.Regex = re + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized path specifier: %+v", r, pt) + } + + if caseSensitive := match.GetCaseSensitive(); caseSensitive != nil { + route.CaseInsensitive = !caseSensitive.Value + } + + for _, h := range match.GetHeaders() { + var header HeaderMatcher + switch ht := h.GetHeaderMatchSpecifier().(type) { + case *v3routepb.HeaderMatcher_ExactMatch: + header.ExactMatch = &ht.ExactMatch + case *v3routepb.HeaderMatcher_SafeRegexMatch: + regex := ht.SafeRegexMatch.GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, nil, fmt.Errorf("route %+v contains an invalid regex %q", r, regex) + } + header.RegexMatch = re + case *v3routepb.HeaderMatcher_RangeMatch: + header.RangeMatch = &Int64Range{ + Start: ht.RangeMatch.Start, + End: ht.RangeMatch.End, + } + case *v3routepb.HeaderMatcher_PresentMatch: + header.PresentMatch = &ht.PresentMatch + case *v3routepb.HeaderMatcher_PrefixMatch: + header.PrefixMatch = &ht.PrefixMatch + case *v3routepb.HeaderMatcher_SuffixMatch: + header.SuffixMatch = &ht.SuffixMatch + default: + return nil, nil, fmt.Errorf("route %+v has an unrecognized header matcher: %+v", r, ht) + } + header.Name = h.GetName() + invert := h.GetInvertMatch() + header.InvertMatch = &invert + route.Headers = append(route.Headers, &header) + } + + if fr := match.GetRuntimeFraction(); fr != nil { + d := fr.GetDefaultValue() + n := d.GetNumerator() + switch d.GetDenominator() { + case v3typepb.FractionalPercent_HUNDRED: + n *= 10000 + case v3typepb.FractionalPercent_TEN_THOUSAND: + n *= 100 + case v3typepb.FractionalPercent_MILLION: + } + route.Fraction = &n + } + + switch r.GetAction().(type) { + case *v3routepb.Route_Route: + route.WeightedClusters = make(map[string]WeightedCluster) + action := r.GetRoute() + + // Hash Policies are only applicable for a Ring Hash LB. + if envconfig.XDSRingHash { + hp, err := hashPoliciesProtoToSlice(action.HashPolicy, logger) + if err != nil { + return nil, nil, err + } + route.HashPolicies = hp + } + + switch a := action.GetClusterSpecifier().(type) { + case *v3routepb.RouteAction_Cluster: + route.WeightedClusters[a.Cluster] = WeightedCluster{Weight: 1} + case *v3routepb.RouteAction_WeightedClusters: + wcs := a.WeightedClusters + var totalWeight uint32 + for _, c := range wcs.Clusters { + w := c.GetWeight().GetValue() + if w == 0 { + continue + } + wc := WeightedCluster{Weight: w} + if !v2 { + cfgs, err := processHTTPFilterOverrides(c.GetTypedPerFilterConfig()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, a, err) + } + wc.HTTPFilterConfigOverride = cfgs + } + route.WeightedClusters[c.GetName()] = wc + totalWeight += w + } + // envoy xds doc + // default TotalWeight https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/route/v3/route_components.proto.html#envoy-v3-api-field-config-route-v3-weightedcluster-total-weight + wantTotalWeight := uint32(100) + if tw := wcs.GetTotalWeight(); tw != nil { + wantTotalWeight = tw.GetValue() + } + if totalWeight != wantTotalWeight { + return nil, nil, fmt.Errorf("route %+v, action %+v, weights of clusters do not add up to total total weight, got: %v, expected total weight from response: %v", r, a, totalWeight, wantTotalWeight) + } + if totalWeight == 0 { + return nil, nil, fmt.Errorf("route %+v, action %+v, has no valid cluster in WeightedCluster action", r, a) + } + case *v3routepb.RouteAction_ClusterHeader: + continue + case *v3routepb.RouteAction_ClusterSpecifierPlugin: + if !envconfig.XDSRLS { + return nil, nil, fmt.Errorf("route %+v, has an unknown ClusterSpecifier: %+v", r, a) + } + if _, ok := csps[a.ClusterSpecifierPlugin]; !ok { + // "When processing RouteActions, if any action includes a + // cluster_specifier_plugin value that is not in + // RouteConfiguration.cluster_specifier_plugins, the + // resource will be NACKed." - RLS in xDS design + return nil, nil, fmt.Errorf("route %+v, action %+v, specifies a cluster specifier plugin %+v that is not in Route Configuration", r, a, a.ClusterSpecifierPlugin) + } + cspNames[a.ClusterSpecifierPlugin] = true + route.ClusterSpecifierPlugin = a.ClusterSpecifierPlugin + default: + return nil, nil, fmt.Errorf("route %+v, has an unknown ClusterSpecifier: %+v", r, a) + } + + msd := action.GetMaxStreamDuration() + // Prefer grpc_timeout_header_max, if set. + dur := msd.GetGrpcTimeoutHeaderMax() + if dur == nil { + dur = msd.GetMaxStreamDuration() + } + if dur != nil { + d := dur.AsDuration() + route.MaxStreamDuration = &d + } + + var err error + route.RetryConfig, err = generateRetryConfig(action.GetRetryPolicy()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v, action %+v: %v", r, action, err) + } + + route.ActionType = RouteActionRoute + + case *v3routepb.Route_NonForwardingAction: + // Expected to be used on server side. + route.ActionType = RouteActionNonForwardingAction + default: + route.ActionType = RouteActionUnsupported + } + + if !v2 { + cfgs, err := processHTTPFilterOverrides(r.GetTypedPerFilterConfig()) + if err != nil { + return nil, nil, fmt.Errorf("route %+v: %v", r, err) + } + route.HTTPFilterConfigOverride = cfgs + } + routesRet = append(routesRet, &route) + } + return routesRet, cspNames, nil +} + +func hashPoliciesProtoToSlice(policies []*v3routepb.RouteAction_HashPolicy, logger *grpclog.PrefixLogger) ([]*HashPolicy, error) { + var hashPoliciesRet []*HashPolicy + for _, p := range policies { + policy := HashPolicy{Terminal: p.Terminal} + switch p.GetPolicySpecifier().(type) { + case *v3routepb.RouteAction_HashPolicy_Header_: + policy.HashPolicyType = HashPolicyTypeHeader + policy.HeaderName = p.GetHeader().GetHeaderName() + if rr := p.GetHeader().GetRegexRewrite(); rr != nil { + regex := rr.GetPattern().GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return nil, fmt.Errorf("hash policy %+v contains an invalid regex %q", p, regex) + } + policy.Regex = re + policy.RegexSubstitution = rr.GetSubstitution() + } + case *v3routepb.RouteAction_HashPolicy_FilterState_: + if p.GetFilterState().GetKey() != "io.grpc.channel_id" { + logger.Infof("hash policy %+v contains an invalid key for filter state policy %q", p, p.GetFilterState().GetKey()) + continue + } + policy.HashPolicyType = HashPolicyTypeChannelID + default: + logger.Infof("hash policy %T is an unsupported hash policy", p.GetPolicySpecifier()) + continue + } + + hashPoliciesRet = append(hashPoliciesRet, &policy) + } + return hashPoliciesRet, nil +} diff --git a/xds/client/resource/version/version.go b/xds/client/resource/version/version.go new file mode 100644 index 0000000000..edfa68762f --- /dev/null +++ b/xds/client/resource/version/version.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package version defines constants to distinguish between supported xDS API +// versions. +package version + +// TransportAPI refers to the API version for xDS transport protocol. This +// describes the xDS gRPC endpoint and version of DiscoveryRequest/Response used +// on the wire. +type TransportAPI int + +const ( + // TransportV2 refers to the v2 xDS transport protocol. + TransportV2 TransportAPI = iota + // TransportV3 refers to the v3 xDS transport protocol. + TransportV3 +) + +// Resource URLs. We need to be able to accept either version of the resource +// regardless of the version of the transport protocol in use. +const ( + googleapiPrefix = "type.googleapis.com/" + + V2ListenerType = "envoy.api.v2.Listener" + V2RouteConfigType = "envoy.api.v2.RouteConfiguration" + V2ClusterType = "envoy.api.v2.Cluster" + V2EndpointsType = "envoy.api.v2.ClusterLoadAssignment" + + V2ListenerURL = googleapiPrefix + V2ListenerType + V2RouteConfigURL = googleapiPrefix + V2RouteConfigType + V2ClusterURL = googleapiPrefix + V2ClusterType + V2EndpointsURL = googleapiPrefix + V2EndpointsType + V2HTTPConnManagerURL = googleapiPrefix + "envoy.config.filter.network.http_connection_manager.v2.HttpConnectionManager" + + V3ListenerType = "envoy.config.listener.v3.Listener" + V3RouteConfigType = "envoy.config.route.v3.RouteConfiguration" + V3ClusterType = "envoy.config.cluster.v3.Cluster" + V3EndpointsType = "envoy.config.endpoint.v3.ClusterLoadAssignment" + + V3ListenerURL = googleapiPrefix + V3ListenerType + V3RouteConfigURL = googleapiPrefix + V3RouteConfigType + V3ClusterURL = googleapiPrefix + V3ClusterType + V3EndpointsURL = googleapiPrefix + V3EndpointsType + V3HTTPConnManagerURL = googleapiPrefix + "envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager" + V3UpstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext" + V3DownstreamTLSContextURL = googleapiPrefix + "envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext" +) diff --git a/xds/client/singleton.go b/xds/client/singleton.go new file mode 100644 index 0000000000..0a7ee3ca19 --- /dev/null +++ b/xds/client/singleton.go @@ -0,0 +1,201 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package client + +import ( + "bytes" + "encoding/json" + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" +) + +const ( + defaultWatchExpiryTimeout = 15 * time.Second + defaultIdleAuthorityDeleteTimeout = 5 * time.Minute +) + +// This is the Client returned by New(). It contains one client implementation, +// and maintains the refcount. +var singletonClient = &clientRefCounted{} + +// To override in tests. +var bootstrapNewConfig = bootstrap.NewConfig + +// clientRefCounted is ref-counted, and to be shared by the xds resolver and +// balancer implementations, across multiple ClientConns and Servers. +type clientRefCounted struct { + *clientImpl + + // This mu protects all the fields, including the embedded clientImpl above. + mu sync.Mutex + refCount int +} + +// New returns a new xdsClient configured by the bootstrap file specified in env +// variable GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG. +// +// The returned xdsClient is a singleton. This function creates the xds client +// if it doesn't already exist. +// +// Note that the first invocation of New() or NewWithConfig() sets the client +// singleton. The following calls will return the singleton xds client without +// checking or using the config. +func New() (XDSClient, error) { + // This cannot just return newRefCounted(), because in error cases, the + // returned nil is a typed nil (*clientRefCounted), which may cause nil + // checks fail. + c, err := newRefCounted() + if err != nil { + return nil, err + } + return c, nil +} + +func newRefCounted() (*clientRefCounted, error) { + singletonClient.mu.Lock() + defer singletonClient.mu.Unlock() + // If the client implementation was created, increment ref count and return + // the client. + if singletonClient.clientImpl != nil { + singletonClient.refCount++ + return singletonClient, nil + } + + // Create the new client implementation. + config, err := bootstrapNewConfig() + if err != nil { + return nil, fmt.Errorf("xds: failed to read bootstrap file: %v", err) + } + c, err := newWithConfig(config, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, err + } + + singletonClient.clientImpl = c + singletonClient.refCount++ + return singletonClient, nil +} + +// NewWithConfig returns a new xdsClient configured by the given config. +// +// The returned xdsClient is a singleton. This function creates the xds client +// if it doesn't already exist. +// +// Note that the first invocation of New() or NewWithConfig() sets the client +// singleton. The following calls will return the singleton xds client without +// checking or using the config. +// +// This function is internal only, for c2p resolver and testing to use. DO NOT +// use this elsewhere. Use New() instead. +func NewWithConfig(config *bootstrap.Config) (XDSClient, error) { + singletonClient.mu.Lock() + defer singletonClient.mu.Unlock() + // If the client implementation was created, increment ref count and return + // the client. + if singletonClient.clientImpl != nil { + singletonClient.refCount++ + return singletonClient, nil + } + + // Create the new client implementation. + c, err := newWithConfig(config, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, err + } + + singletonClient.clientImpl = c + singletonClient.refCount++ + return singletonClient, nil +} + +// Close closes the client. It does ref count of the xds client implementation, +// and closes the gRPC connection to the management server when ref count +// reaches 0. +func (c *clientRefCounted) Close() { + c.mu.Lock() + defer c.mu.Unlock() + c.refCount-- + if c.refCount == 0 { + c.clientImpl.Close() + // Set clientImpl back to nil. So if New() is called after this, a new + // implementation will be created. + c.clientImpl = nil + } +} + +// NewWithConfigForTesting is exported for testing only. +// +// Note that this function doesn't set the singleton, so that the testing states +// don't leak. +func NewWithConfigForTesting(config *bootstrap.Config, watchExpiryTimeout time.Duration) (XDSClient, error) { + cl, err := newWithConfig(config, watchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, err + } + return &clientRefCounted{clientImpl: cl, refCount: 1}, nil +} + +// NewClientWithBootstrapContents returns an xds client for this config, +// separate from the global singleton. This should be used for testing +// purposes only. +func NewClientWithBootstrapContents(contents []byte) (XDSClient, error) { + // Normalize the contents + buf := bytes.Buffer{} + err := json.Indent(&buf, contents, "", "") + if err != nil { + return nil, fmt.Errorf("xds: error normalizing JSON: %v", err) + } + contents = bytes.TrimSpace(buf.Bytes()) + + clientsMu.Lock() + defer clientsMu.Unlock() + if c := clients[string(contents)]; c != nil { + c.mu.Lock() + // Since we don't remove the *Client from the map when it is closed, we + // need to recreate the impl if the ref count dropped to zero. + if c.refCount > 0 { + c.refCount++ + c.mu.Unlock() + return c, nil + } + c.mu.Unlock() + } + + bcfg, err := bootstrap.NewConfigFromContents(contents) + if err != nil { + return nil, fmt.Errorf("xds: error with bootstrap config: %v", err) + } + + cImpl, err := newWithConfig(bcfg, defaultWatchExpiryTimeout, defaultIdleAuthorityDeleteTimeout) + if err != nil { + return nil, err + } + + c := &clientRefCounted{clientImpl: cImpl, refCount: 1} + clients[string(contents)] = c + return c, nil +} + +var ( + clients = map[string]*clientRefCounted{} + clientsMu sync.Mutex +) diff --git a/xds/client/watchers.go b/xds/client/watchers.go new file mode 100644 index 0000000000..57f6506c73 --- /dev/null +++ b/xds/client/watchers.go @@ -0,0 +1,105 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// WatchListener uses LDS to discover information about the provided listener. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +func (c *clientImpl) WatchListener(serviceName string, cb func(resource.ListenerUpdate, error)) (cancel func()) { + n := resource.ParseName(serviceName) + a, unref, err := c.findAuthority(n) + if err != nil { + cb(resource.ListenerUpdate{}, err) + return func() {} + } + cancelF := a.watchListener(n.String(), cb) + return func() { + cancelF() + unref() + } +} + +// WatchRouteConfig starts a listener watcher for the service. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +func (c *clientImpl) WatchRouteConfig(routeName string, cb func(resource.RouteConfigUpdate, error)) (cancel func()) { + n := resource.ParseName(routeName) + a, unref, err := c.findAuthority(n) + if err != nil { + cb(resource.RouteConfigUpdate{}, err) + return func() {} + } + cancelF := a.watchRouteConfig(n.String(), cb) + return func() { + cancelF() + unref() + } +} + +// WatchCluster uses CDS to discover information about the provided +// clusterName. +// +// WatchCluster can be called multiple times, with same or different +// clusterNames. Each call will start an independent watcher for the resource. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +func (c *clientImpl) WatchCluster(clusterName string, cb func(resource.ClusterUpdate, error)) (cancel func()) { + n := resource.ParseName(clusterName) + a, unref, err := c.findAuthority(n) + if err != nil { + cb(resource.ClusterUpdate{}, err) + return func() {} + } + cancelF := a.watchCluster(n.String(), cb) + return func() { + cancelF() + unref() + } +} + +// WatchEndpoints uses EDS to discover endpoints in the provided clusterName. +// +// WatchEndpoints can be called multiple times, with same or different +// clusterNames. Each call will start an independent watcher for the resource. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +func (c *clientImpl) WatchEndpoints(clusterName string, cb func(resource.EndpointsUpdate, error)) (cancel func()) { + n := resource.ParseName(clusterName) + a, unref, err := c.findAuthority(n) + if err != nil { + cb(resource.EndpointsUpdate{}, err) + return func() {} + } + cancelF := a.watchEndpoints(n.String(), cb) + return func() { + cancelF() + unref() + } +} diff --git a/xds/clusterspecifier/cluster_specifier.go b/xds/clusterspecifier/cluster_specifier.go new file mode 100644 index 0000000000..54776f20cf --- /dev/null +++ b/xds/clusterspecifier/cluster_specifier.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package clusterspecifier contains the ClusterSpecifier interface and a registry for +// storing and retrieving their implementations. +package clusterspecifier + +import ( + "github.com/golang/protobuf/proto" +) + +// BalancerConfig is the Go Native JSON representation of a balancer +// configuration. +type BalancerConfig []map[string]interface{} + +// ClusterSpecifier defines the parsing functionality of a Cluster Specifier. +type ClusterSpecifier interface { + // TypeURLs are the proto message types supported by this + // ClusterSpecifierPlugin. A ClusterSpecifierPlugin will be registered by + // each of its supported message types. + TypeURLs() []string + // ParseClusterSpecifierConfig parses the provided configuration + // proto.Message from the top level RDS configuration. The resulting + // BalancerConfig will be used as configuration for a child LB Policy of the + // Cluster Manager LB Policy. + ParseClusterSpecifierConfig(proto.Message) (BalancerConfig, error) +} + +var ( + // m is a map from scheme to filter. + m = make(map[string]ClusterSpecifier) +) + +// Register registers the ClusterSpecifierPlugin to the ClusterSpecifier map. +// cs.TypeURLs() will be used as the types for this ClusterSpecifierPlugin. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple cluster specifier +// plugins are registered with the same type URL, the one registered last will +// take effect. +func Register(cs ClusterSpecifier) { + for _, u := range cs.TypeURLs() { + m[u] = cs + } +} + +// Get returns the ClusterSpecifier registered with typeURL. +// +// If no cluster specifier is registered with typeURL, nil will be returned. +func Get(typeURL string) ClusterSpecifier { + return m[typeURL] +} diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go new file mode 100644 index 0000000000..f6f903824b --- /dev/null +++ b/xds/httpfilter/fault/fault.go @@ -0,0 +1,301 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package fault implements the Envoy Fault Injection HTTP filter. +package fault + +import ( + "context" + "errors" + "fmt" + "io" + "strconv" + "sync/atomic" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +) + +const headerAbortHTTPStatus = "x-envoy-fault-abort-request" +const headerAbortGRPCStatus = "x-envoy-fault-abort-grpc-request" +const headerAbortPercentage = "x-envoy-fault-abort-request-percentage" + +const headerDelayPercentage = "x-envoy-fault-delay-request-percentage" +const headerDelayDuration = "x-envoy-fault-delay-request" + +var statusMap = map[int]codes.Code{ + 400: codes.Internal, + 401: codes.Unauthenticated, + 403: codes.PermissionDenied, + 404: codes.Unimplemented, + 429: codes.Unavailable, + 502: codes.Unavailable, + 503: codes.Unavailable, + 504: codes.Unavailable, +} + +func init() { + httpfilter.Register(builder{}) +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + config *fpb.HTTPFault +} + +func (builder) TypeURLs() []string { + return []string{"type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault"} +} + +// Parsing is the same for the base config and the override config. +func parseConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("fault: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(fpb.HTTPFault) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("fault: error parsing config %v: %v", cfg, err) + } + return config{config: msg}, nil +} + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + return parseConfig(cfg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + return parseConfig(override) +} + +func (builder) IsTerminal() bool { + return false +} + +var _ httpfilter.ClientInterceptorBuilder = builder{} + +func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("fault: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("fault: incorrect override config type provided (%T): %v", override, override) + } + } + + icfg := c.config + if (icfg.GetMaxActiveFaults() != nil && icfg.GetMaxActiveFaults().GetValue() == 0) || + (icfg.GetDelay() == nil && icfg.GetAbort() == nil) { + return nil, nil + } + return &interceptor{config: icfg}, nil +} + +type interceptor struct { + config *fpb.HTTPFault +} + +var activeFaults uint32 // global active faults; accessed atomically + +func (i *interceptor) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) { + if maxAF := i.config.GetMaxActiveFaults(); maxAF != nil { + defer atomic.AddUint32(&activeFaults, ^uint32(0)) // decrement counter + if af := atomic.AddUint32(&activeFaults, 1); af > maxAF.GetValue() { + // Would exceed maximum active fault limit. + return newStream(ctx, done) + } + } + + if err := injectDelay(ctx, i.config.GetDelay()); err != nil { + return nil, err + } + + if err := injectAbort(ctx, i.config.GetAbort()); err != nil { + if err == errOKStream { + return &okStream{ctx: ctx}, nil + } + return nil, err + } + return newStream(ctx, done) +} + +// For overriding in tests +var randIntn = grpcrand.Intn +var newTimer = time.NewTimer + +func injectDelay(ctx context.Context, delayCfg *cpb.FaultDelay) error { + numerator, denominator := splitPct(delayCfg.GetPercentage()) + var delay time.Duration + switch delayType := delayCfg.GetFaultDelaySecifier().(type) { + case *cpb.FaultDelay_FixedDelay: + delay = delayType.FixedDelay.AsDuration() + case *cpb.FaultDelay_HeaderDelay_: + md, _ := metadata.FromOutgoingContext(ctx) + v := md[headerDelayDuration] + if v == nil { + // No delay configured for this RPC. + return nil + } + ms, ok := parseIntFromMD(v) + if !ok { + // Malformed header; no delay. + return nil + } + delay = time.Duration(ms) * time.Millisecond + if v := md[headerDelayPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if delay == 0 || randIntn(denominator) >= numerator { + return nil + } + t := newTimer(delay) + select { + case <-t.C: + case <-ctx.Done(): + t.Stop() + return ctx.Err() + } + return nil +} + +func injectAbort(ctx context.Context, abortCfg *fpb.FaultAbort) error { + numerator, denominator := splitPct(abortCfg.GetPercentage()) + code := codes.OK + okCode := false + switch errType := abortCfg.GetErrorType().(type) { + case *fpb.FaultAbort_HttpStatus: + code, okCode = grpcFromHTTP(int(errType.HttpStatus)) + case *fpb.FaultAbort_GrpcStatus: + code, okCode = sanitizeGRPCCode(codes.Code(errType.GrpcStatus)), true + case *fpb.FaultAbort_HeaderAbort_: + md, _ := metadata.FromOutgoingContext(ctx) + if v := md[headerAbortHTTPStatus]; v != nil { + // HTTP status has priority over gRPC status. + if httpStatus, ok := parseIntFromMD(v); ok { + code, okCode = grpcFromHTTP(httpStatus) + } + } else if v := md[headerAbortGRPCStatus]; v != nil { + if grpcStatus, ok := parseIntFromMD(v); ok { + code, okCode = sanitizeGRPCCode(codes.Code(grpcStatus)), true + } + } + if v := md[headerAbortPercentage]; v != nil { + if num, ok := parseIntFromMD(v); ok && num < numerator { + numerator = num + } + } + } + if !okCode || randIntn(denominator) >= numerator { + return nil + } + if code == codes.OK { + return errOKStream + } + return status.Errorf(code, "RPC terminated due to fault injection") +} + +var errOKStream = errors.New("stream terminated early with OK status") + +// parseIntFromMD returns the integer in the last header or nil if parsing +// failed. +func parseIntFromMD(header []string) (int, bool) { + if len(header) == 0 { + return 0, false + } + v, err := strconv.Atoi(header[len(header)-1]) + return v, err == nil +} + +func splitPct(fp *tpb.FractionalPercent) (num int, den int) { + if fp == nil { + return 0, 100 + } + num = int(fp.GetNumerator()) + switch fp.GetDenominator() { + case tpb.FractionalPercent_HUNDRED: + return num, 100 + case tpb.FractionalPercent_TEN_THOUSAND: + return num, 10 * 1000 + case tpb.FractionalPercent_MILLION: + return num, 1000 * 1000 + } + return num, 100 +} + +func grpcFromHTTP(httpStatus int) (codes.Code, bool) { + if httpStatus < 200 || httpStatus >= 600 { + // Malformed; ignore this fault type. + return codes.OK, false + } + if c := statusMap[httpStatus]; c != codes.OK { + // OK = 0/the default for the map. + return c, true + } + // All undefined HTTP status codes convert to Unknown. HTTP status of 200 + // is "success", but gRPC converts to Unknown due to missing grpc status. + return codes.Unknown, true +} + +func sanitizeGRPCCode(c codes.Code) codes.Code { + if c > codes.Code(16) { + return codes.Unknown + } + return c +} + +type okStream struct { + ctx context.Context +} + +func (*okStream) Header() (metadata.MD, error) { return nil, nil } +func (*okStream) Trailer() metadata.MD { return nil } +func (*okStream) CloseSend() error { return nil } +func (o *okStream) Context() context.Context { return o.ctx } +func (*okStream) SendMsg(m interface{}) error { return io.EOF } +func (*okStream) RecvMsg(m interface{}) error { return io.EOF } diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go new file mode 100644 index 0000000000..4f167590cd --- /dev/null +++ b/xds/httpfilter/httpfilter.go @@ -0,0 +1,108 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package httpfilter contains the HTTPFilter interface and a registry for +// storing and retrieving their implementations. +package httpfilter + +import ( + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "github.com/golang/protobuf/proto" +) + +// FilterConfig represents an opaque data structure holding configuration for a +// filter. Embed this interface to implement it. +type FilterConfig interface { + isFilterConfig() +} + +// Filter defines the parsing functionality of an HTTP filter. A Filter may +// optionally implement either ClientInterceptorBuilder or +// ServerInterceptorBuilder or both, indicating it is capable of working on the +// client side or server side or both, respectively. +type Filter interface { + // TypeURLs are the proto message types supported by this filter. A filter + // will be registered by each of its supported message types. + TypeURLs() []string + // ParseFilterConfig parses the provided configuration proto.Message from + // the LDS configuration of this filter. This may be an anypb.Any, a + // udpa.type.v1.TypedStruct, or an xds.type.v3.TypedStruct for filters that + // do not accept a custom type. The resulting FilterConfig will later be + // passed to Build. + ParseFilterConfig(proto.Message) (FilterConfig, error) + // ParseFilterConfigOverride parses the provided override configuration + // proto.Message from the RDS override configuration of this filter. This + // may be an anypb.Any, a udpa.type.v1.TypedStruct, or an + // xds.type.v3.TypedStruct for filters that do not accept a custom type. + // The resulting FilterConfig will later be passed to Build. + ParseFilterConfigOverride(proto.Message) (FilterConfig, error) + // IsTerminal returns whether this Filter is terminal or not (i.e. it must + // be last filter in the filter chain). + IsTerminal() bool +} + +// ClientInterceptorBuilder constructs a Client Interceptor. If this type is +// implemented by a Filter, it is capable of working on a client. +type ClientInterceptorBuilder interface { + // BuildClientInterceptor uses the FilterConfigs produced above to produce + // an HTTP filter interceptor for clients. config will always be non-nil, + // but override may be nil if no override config exists for the filter. It + // is valid for Build to return a nil Interceptor and a nil error. In this + // case, the RPC will not be intercepted by this filter. + BuildClientInterceptor(config, override FilterConfig) (iresolver.ClientInterceptor, error) +} + +// ServerInterceptorBuilder constructs a Server Interceptor. If this type is +// implemented by a Filter, it is capable of working on a server. +type ServerInterceptorBuilder interface { + // BuildServerInterceptor uses the FilterConfigs produced above to produce + // an HTTP filter interceptor for servers. config will always be non-nil, + // but override may be nil if no override config exists for the filter. It + // is valid for Build to return a nil Interceptor and a nil error. In this + // case, the RPC will not be intercepted by this filter. + BuildServerInterceptor(config, override FilterConfig) (iresolver.ServerInterceptor, error) +} + +var ( + // m is a map from scheme to filter. + m = make(map[string]Filter) +) + +// Register registers the HTTP filter Builder to the filter map. b.TypeURLs() +// will be used as the types for this filter. +// +// NOTE: this function must only be called during initialization time (i.e. in +// an init() function), and is not thread-safe. If multiple filters are +// registered with the same type URL, the one registered last will take effect. +func Register(b Filter) { + for _, u := range b.TypeURLs() { + m[u] = b + } +} + +// UnregisterForTesting unregisters the HTTP Filter for testing purposes. +func UnregisterForTesting(typeURL string) { + delete(m, typeURL) +} + +// Get returns the HTTPFilter registered with typeURL. +// +// If no filter is register with typeURL, nil will be returned. +func Get(typeURL string) Filter { + return m[typeURL] +} diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go new file mode 100644 index 0000000000..eacf7361c6 --- /dev/null +++ b/xds/httpfilter/rbac/rbac.go @@ -0,0 +1,220 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package rbac implements the Envoy RBAC HTTP filter. +package rbac + +import ( + "context" + "errors" + "fmt" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/rbac" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" + + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +) + +func init() { + if envconfig.XDSRBAC { + httpfilter.Register(builder{}) + } +} + +// RegisterForTesting registers the RBAC HTTP Filter for testing purposes, regardless +// of the RBAC environment variable. This is needed because there is no way to set the RBAC +// environment variable to true in a test before init() in this package is run. +func RegisterForTesting() { + httpfilter.Register(builder{}) +} + +// UnregisterForTesting unregisters the RBAC HTTP Filter for testing purposes. This is needed because +// there is no way to unregister the HTTP Filter after registering it solely for testing purposes using +// rbac.RegisterForTesting() +func UnregisterForTesting() { + for _, typeURL := range builder.TypeURLs(builder{}) { + httpfilter.UnregisterForTesting(typeURL) + } +} + +type builder struct { +} + +type config struct { + httpfilter.FilterConfig + chainEngine *rbac.ChainEngine +} + +func (builder) TypeURLs() []string { + return []string{ + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBAC", + "type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute", + } +} + +// Parsing is the same for the base config and the override config. +func parseConfig(rbacCfg *rpb.RBAC) (httpfilter.FilterConfig, error) { + // All the validation logic described in A41. + for _, policy := range rbacCfg.GetRules().GetPolicies() { + // "Policy.condition and Policy.checked_condition must cause a + // validation failure if present." - A41 + if policy.Condition != nil { + return nil, errors.New("rbac: Policy.condition is present") + } + if policy.CheckedCondition != nil { + return nil, errors.New("rbac: policy.CheckedCondition is present") + } + + // "It is also a validation failure if Permission or Principal has a + // header matcher for a grpc- prefixed header name or :scheme." - A41 + for _, principal := range policy.Principals { + name := principal.GetHeader().GetName() + if name == ":scheme" || strings.HasPrefix(name, "grpc-") { + return nil, fmt.Errorf("rbac: principal header matcher for %v is :scheme or starts with grpc", name) + } + } + for _, permission := range policy.Permissions { + name := permission.GetHeader().GetName() + if name == ":scheme" || strings.HasPrefix(name, "grpc-") { + return nil, fmt.Errorf("rbac: permission header matcher for %v is :scheme or starts with grpc", name) + } + } + } + + // "Envoy aliases :authority and Host in its header map implementation, so + // they should be treated equivalent for the RBAC matchers; there must be no + // behavior change depending on which of the two header names is used in the + // RBAC policy." - A41. Loop through config's principals and policies, change + // any header matcher with value "host" to :authority", as that is what + // grpc-go shifts both headers to in transport layer. + for _, policy := range rbacCfg.GetRules().GetPolicies() { + for _, principal := range policy.Principals { + if principal.GetHeader().GetName() == "host" { + principal.GetHeader().Name = ":authority" + } + } + for _, permission := range policy.Permissions { + if permission.GetHeader().GetName() == "host" { + permission.GetHeader().Name = ":authority" + } + } + } + + // Two cases where this HTTP Filter is a no op: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if rbacCfg.Rules == nil || rbacCfg.GetRules().GetAction() == v3rbacpb.RBAC_LOG { + return config{}, nil + } + + ce, err := rbac.NewChainEngine([]*v3rbacpb.RBAC{rbacCfg.GetRules()}) + if err != nil { + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if rbacCfg.GetRules().GetAction() != v3rbacpb.RBAC_LOG { + return nil, fmt.Errorf("rbac: error constructing matching engine: %v", err) + } + } + + return config{chainEngine: ce}, nil +} + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(rpb.RBAC) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing config %v: %v", cfg, err) + } + return parseConfig(msg) +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + if override == nil { + return nil, fmt.Errorf("rbac: nil configuration message provided") + } + any, ok := override.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("rbac: error parsing override config %v: unknown type %T", override, override) + } + msg := new(rpb.RBACPerRoute) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("rbac: error parsing override config %v: %v", override, err) + } + return parseConfig(msg.Rbac) +} + +func (builder) IsTerminal() bool { + return false +} + +var _ httpfilter.ServerInterceptorBuilder = builder{} + +// BuildServerInterceptor is an optional interface builder implements in order +// to signify it works server side. +func (builder) BuildServerInterceptor(cfg httpfilter.FilterConfig, override httpfilter.FilterConfig) (resolver.ServerInterceptor, error) { + if cfg == nil { + return nil, fmt.Errorf("rbac: nil config provided") + } + + c, ok := cfg.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect config type provided (%T): %v", cfg, cfg) + } + + if override != nil { + // override completely replaces the listener configuration; but we + // still validate the listener config type. + c, ok = override.(config) + if !ok { + return nil, fmt.Errorf("rbac: incorrect override config type provided (%T): %v", override, override) + } + } + + // RBAC HTTP Filter is a no op from one of these two cases: + // "If absent, no enforcing RBAC policy will be applied" - RBAC + // Documentation for Rules field. + // "At this time, if the RBAC.action is Action.LOG then the policy will be + // completely ignored, as if RBAC was not configurated." - A41 + if c.chainEngine == nil { + return nil, nil + } + return &interceptor{chainEngine: c.chainEngine}, nil +} + +type interceptor struct { + chainEngine *rbac.ChainEngine +} + +func (i *interceptor) AllowRPC(ctx context.Context) error { + return i.chainEngine.IsAuthorized(ctx) +} diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go new file mode 100644 index 0000000000..78a433a696 --- /dev/null +++ b/xds/httpfilter/router/router.go @@ -0,0 +1,114 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package router implements the Envoy Router HTTP filter. +package router + +import ( + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" + + pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" +) + +// TypeURL is the message type for the Router configuration. +const TypeURL = "type.googleapis.com/envoy.extensions.filters.http.router.v3.Router" + +func init() { + httpfilter.Register(builder{}) +} + +// IsRouterFilter returns true iff a HTTP filter is a Router filter. +func IsRouterFilter(b httpfilter.Filter) bool { + _, ok := b.(builder) + return ok +} + +type builder struct { +} + +func (builder) TypeURLs() []string { return []string{TypeURL} } + +func (builder) ParseFilterConfig(cfg proto.Message) (httpfilter.FilterConfig, error) { + // The gRPC router filter does not currently use any fields from the + // config. Verify type only. + if cfg == nil { + return nil, fmt.Errorf("router: nil configuration message provided") + } + any, ok := cfg.(*anypb.Any) + if !ok { + return nil, fmt.Errorf("router: error parsing config %v: unknown type %T", cfg, cfg) + } + msg := new(pb.Router) + if err := ptypes.UnmarshalAny(any, msg); err != nil { + return nil, fmt.Errorf("router: error parsing config %v: %v", cfg, err) + } + return config{}, nil +} + +func (builder) ParseFilterConfigOverride(override proto.Message) (httpfilter.FilterConfig, error) { + if override != nil { + return nil, fmt.Errorf("router: unexpected config override specified: %v", override) + } + return config{}, nil +} + +func (builder) IsTerminal() bool { + return true +} + +var ( + _ httpfilter.ClientInterceptorBuilder = builder{} + _ httpfilter.ServerInterceptorBuilder = builder{} +) + +func (builder) BuildClientInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ClientInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is implemented within the xds resolver's config + // selector, not as a separate plugin. So we return a nil HTTPFilter, + // which will not be invoked. + return nil, nil +} + +func (builder) BuildServerInterceptor(cfg, override httpfilter.FilterConfig) (iresolver.ServerInterceptor, error) { + if _, ok := cfg.(config); !ok { + return nil, fmt.Errorf("router: incorrect config type provided (%T): %v", cfg, cfg) + } + if override != nil { + return nil, fmt.Errorf("router: unexpected override configuration specified: %v", override) + } + // The gRPC router is currently unimplemented on the server side. So we + // return a nil HTTPFilter, which will not be invoked. + return nil, nil +} + +// The gRPC router filter does not currently support any configuration. Verify +// type only. +type config struct { + httpfilter.FilterConfig +} diff --git a/xds/internal.go b/xds/internal.go new file mode 100644 index 0000000000..1b596bf357 --- /dev/null +++ b/xds/internal.go @@ -0,0 +1,88 @@ +/* + * Copyright 2016 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package internal contains gRPC-internal code, to avoid polluting +// the godoc of the top-level grpc package. It must not import any grpc +// symbols to avoid circular dependencies. +package internal + +import ( + "context" + "time" + + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/serviceconfig" +) + +var ( + // WithHealthCheckFunc is set by dialoptions.go + WithHealthCheckFunc interface{} // func (HealthChecker) DialOption + // HealthCheckFunc is used to provide client-side LB channel health checking + HealthCheckFunc HealthChecker + // BalancerUnregister is exported by package balancer to unregister a balancer. + BalancerUnregister func(name string) + // KeepaliveMinPingTime is the minimum ping interval. This must be 10s by + // default, but tests may wish to set it lower for convenience. + KeepaliveMinPingTime = 10 * time.Second + // ParseServiceConfigForTesting is for creating a fake + // ClientConn for resolver testing only + ParseServiceConfigForTesting interface{} // func(string) *serviceconfig.ParseResult + // EqualServiceConfigForTesting is for testing service config generation and + // parsing. Both a and b should be returned by ParseServiceConfigForTesting. + // This function compares the config without rawJSON stripped, in case the + // there's difference in white space. + EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool + // GetCertificateProviderBuilder returns the registered builder for the + // given name. This is set by package certprovider for use from xDS + // bootstrap code while parsing certificate provider configs in the + // bootstrap file. + GetCertificateProviderBuilder interface{} // func(string) certprovider.Builder + // GetXDSHandshakeInfoForTesting returns a pointer to the xds.HandshakeInfo + // stored in the passed in attributes. This is set by + // credentials/xds/xds.go. + GetXDSHandshakeInfoForTesting interface{} // func (*attributes.Attributes) *xds.HandshakeInfo + // GetServerCredentials returns the transport credentials configured on a + // gRPC server. An xDS-enabled server needs to know what type of credentials + // is configured on the underlying gRPC server. This is set by server.go. + GetServerCredentials interface{} // func (*grpc.Server) credentials.TransportCredentials + // DrainServerTransports initiates a graceful close of existing connections + // on a gRPC server accepted on the provided listener address. An + // xDS-enabled server invokes this method on a grpc.Server when a particular + // listener moves to "not-serving" mode. + DrainServerTransports interface{} // func(*grpc.Server, string) +) + +// HealthChecker defines the signature of the client-side LB channel health checking function. +// +// The implementation is expected to create a health checking RPC stream by +// calling newStream(), watch for the health status of serviceName, and report +// it's health back by calling setConnectivityState(). +// +// The health checking protocol is defined at: +// https://github.com/grpc/grpc/blob/master/doc/health-checking.md +type HealthChecker func(ctx context.Context, newStream func(string) (interface{}, error), setConnectivityState func(connectivity.State, error), serviceName string) error + +const ( + // CredsBundleModeFallback switches GoogleDefaultCreds to fallback mode. + CredsBundleModeFallback = "fallback" + // CredsBundleModeBalancer switches GoogleDefaultCreds to grpclb balancer + // mode. + CredsBundleModeBalancer = "balancer" + // CredsBundleModeBackendFromBalancer switches GoogleDefaultCreds to mode + // that supports backend returned by grpclb balancer. + CredsBundleModeBackendFromBalancer = "backend-from-balancer" +) diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go new file mode 100644 index 0000000000..1c3a947b0f --- /dev/null +++ b/xds/resolver/logging.go @@ -0,0 +1,34 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/grpclog" +) + +const prefix = "[xds-resolver %p] " + +var logger = grpclog.Component("xds") + +func prefixLogger(p *xdsResolver) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(prefix, p)) +} diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go new file mode 100644 index 0000000000..4640662c0d --- /dev/null +++ b/xds/resolver/serviceconfig.go @@ -0,0 +1,440 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "context" + "encoding/json" + "fmt" + "math/bits" + "strings" + "sync/atomic" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter/router" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" + xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const ( + cdsName = "cds_experimental" + xdsClusterManagerName = "xds_cluster_manager_experimental" + clusterPrefix = "cluster:" + clusterSpecifierPluginPrefix = "cluster_specifier_plugin:" +) + +type serviceConfig struct { + LoadBalancingConfig balancerConfig `json:"loadBalancingConfig"` +} + +type balancerConfig []map[string]interface{} + +func newBalancerConfig(name string, config interface{}) balancerConfig { + return []map[string]interface{}{{name: config}} +} + +type cdsBalancerConfig struct { + Cluster string `json:"cluster"` +} + +type xdsChildConfig struct { + ChildPolicy balancerConfig `json:"childPolicy"` +} + +type xdsClusterManagerConfig struct { + Children map[string]xdsChildConfig `json:"children"` +} + +// pruneActiveClusters deletes entries in r.activeClusters with zero +// references. +func (r *xdsResolver) pruneActiveClusters() { + for cluster, ci := range r.activeClusters { + if atomic.LoadInt32(&ci.refCount) == 0 { + delete(r.activeClusters, cluster) + } + } +} + +// serviceConfigJSON produces a service config in JSON format representing all +// the clusters referenced in activeClusters. This includes clusters with zero +// references, so they must be pruned first. +func serviceConfigJSON(activeClusters map[string]*clusterInfo) ([]byte, error) { + // Generate children (all entries in activeClusters). + children := make(map[string]xdsChildConfig) + for cluster, ci := range activeClusters { + children[cluster] = ci.cfg + } + + sc := serviceConfig{ + LoadBalancingConfig: newBalancerConfig( + xdsClusterManagerName, xdsClusterManagerConfig{Children: children}, + ), + } + + bs, err := json.Marshal(sc) + if err != nil { + return nil, fmt.Errorf("failed to marshal json: %v", err) + } + return bs, nil +} + +type virtualHost struct { + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig + // retry policy present in virtual host + retryConfig *resource.RetryConfig +} + +// routeCluster holds information about a cluster as referenced by a route. +type routeCluster struct { + name string + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig +} + +type route struct { + m *resource.CompositeMatcher // converted from route matchers + clusters wrr.WRR // holds *routeCluster entries + maxStreamDuration time.Duration + // map from filter name to its config + httpFilterConfigOverride map[string]httpfilter.FilterConfig + retryConfig *resource.RetryConfig + hashPolicies []*resource.HashPolicy +} + +func (r route) String() string { + return fmt.Sprintf("%s -> { clusters: %v, maxStreamDuration: %v }", r.m.String(), r.clusters, r.maxStreamDuration) +} + +type configSelector struct { + r *xdsResolver + virtualHost virtualHost + routes []route + clusters map[string]*clusterInfo + httpFilterConfig []resource.HTTPFilter +} + +var errNoMatchedRouteFound = status.Errorf(codes.Unavailable, "no matched route was found") + +func (cs *configSelector) SelectConfig(rpcInfo iresolver.RPCInfo) (*iresolver.RPCConfig, error) { + if cs == nil { + return nil, status.Errorf(codes.Unavailable, "no valid clusters") + } + var rt *route + // Loop through routes in order and select first match. + for _, r := range cs.routes { + if r.m.Match(rpcInfo) { + rt = &r + break + } + } + if rt == nil || rt.clusters == nil { + return nil, errNoMatchedRouteFound + } + + cluster, ok := rt.clusters.Next().(*routeCluster) + if !ok { + return nil, status.Errorf(codes.Internal, "error retrieving cluster for match: %v (%T)", cluster, cluster) + } + + // Add a ref to the selected cluster, as this RPC needs this cluster until + // it is committed. + ref := &cs.clusters[cluster.name].refCount + atomic.AddInt32(ref, 1) + + interceptor, err := cs.newInterceptor(rt, cluster) + if err != nil { + return nil, err + } + + lbCtx := clustermanager.SetPickedCluster(rpcInfo.Context, cluster.name) + // Request Hashes are only applicable for a Ring Hash LB. + if envconfig.XDSRingHash { + lbCtx = ringhash.SetRequestHash(lbCtx, cs.generateHash(rpcInfo, rt.hashPolicies)) + } + + config := &iresolver.RPCConfig{ + // Communicate to the LB policy the chosen cluster and request hash, if Ring Hash LB policy. + Context: lbCtx, + OnCommitted: func() { + // When the RPC is committed, the cluster is no longer required. + // Decrease its ref. + if v := atomic.AddInt32(ref, -1); v == 0 { + // This entry will be removed from activeClusters when + // producing the service config for the empty update. + select { + case cs.r.updateCh <- suWithError{emptyUpdate: true}: + default: + } + } + }, + Interceptor: interceptor, + } + + if rt.maxStreamDuration != 0 { + config.MethodConfig.Timeout = &rt.maxStreamDuration + } + if rt.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(rt.retryConfig) + } else if cs.virtualHost.retryConfig != nil { + config.MethodConfig.RetryPolicy = retryConfigToPolicy(cs.virtualHost.retryConfig) + } + + return config, nil +} + +func retryConfigToPolicy(config *resource.RetryConfig) *serviceconfig.RetryPolicy { + return &serviceconfig.RetryPolicy{ + MaxAttempts: int(config.NumRetries) + 1, + InitialBackoff: config.RetryBackoff.BaseInterval, + MaxBackoff: config.RetryBackoff.MaxInterval, + BackoffMultiplier: 2, + RetryableStatusCodes: config.RetryOn, + } +} + +func (cs *configSelector) generateHash(rpcInfo iresolver.RPCInfo, hashPolicies []*resource.HashPolicy) uint64 { + var hash uint64 + var generatedHash bool + for _, policy := range hashPolicies { + var policyHash uint64 + var generatedPolicyHash bool + switch policy.HashPolicyType { + case resource.HashPolicyTypeHeader: + md, ok := metadata.FromOutgoingContext(rpcInfo.Context) + if !ok { + continue + } + values := md.Get(policy.HeaderName) + // If the header isn't present, no-op. + if len(values) == 0 { + continue + } + joinedValues := strings.Join(values, ",") + if policy.Regex != nil { + joinedValues = policy.Regex.ReplaceAllString(joinedValues, policy.RegexSubstitution) + } + policyHash = xxhash.Sum64String(joinedValues) + generatedHash = true + generatedPolicyHash = true + case resource.HashPolicyTypeChannelID: + // Hash the ClientConn pointer which logically uniquely + // identifies the client. + policyHash = xxhash.Sum64String(fmt.Sprintf("%p", &cs.r.cc)) + generatedHash = true + generatedPolicyHash = true + } + + // Deterministically combine the hash policies. Rotating prevents + // duplicate hash policies from cancelling each other out and preserves + // the 64 bits of entropy. + if generatedPolicyHash { + hash = bits.RotateLeft64(hash, 1) + hash = hash ^ policyHash + } + + // If terminal policy and a hash has already been generated, ignore the + // rest of the policies and use that hash already generated. + if policy.Terminal && generatedHash { + break + } + } + + if generatedHash { + return hash + } + // If no generated hash return a random long. In the grand scheme of things + // this logically will map to choosing a random backend to route request to. + return grpcrand.Uint64() +} + +func (cs *configSelector) newInterceptor(rt *route, cluster *routeCluster) (iresolver.ClientInterceptor, error) { + if len(cs.httpFilterConfig) == 0 { + return nil, nil + } + interceptors := make([]iresolver.ClientInterceptor, 0, len(cs.httpFilterConfig)) + for _, filter := range cs.httpFilterConfig { + if router.IsRouterFilter(filter.Filter) { + // Ignore any filters after the router filter. The router itself + // is currently a nop. + return &interceptorList{interceptors: interceptors}, nil + } + override := cluster.httpFilterConfigOverride[filter.Name] // cluster is highest priority + if override == nil { + override = rt.httpFilterConfigOverride[filter.Name] // route is second priority + } + if override == nil { + override = cs.virtualHost.httpFilterConfigOverride[filter.Name] // VH is third & lowest priority + } + ib, ok := filter.Filter.(httpfilter.ClientInterceptorBuilder) + if !ok { + // Should not happen if it passed xdsClient validation. + return nil, fmt.Errorf("filter does not support use in client") + } + i, err := ib.BuildClientInterceptor(filter.Config, override) + if err != nil { + return nil, fmt.Errorf("error constructing filter: %v", err) + } + if i != nil { + interceptors = append(interceptors, i) + } + } + return nil, fmt.Errorf("error in xds config: no router filter present") +} + +// stop decrements refs of all clusters referenced by this config selector. +func (cs *configSelector) stop() { + // The resolver's old configSelector may be nil. Handle that here. + if cs == nil { + return + } + // If any refs drop to zero, we'll need a service config update to delete + // the cluster. + needUpdate := false + // Loops over cs.clusters, but these are pointers to entries in + // activeClusters. + for _, ci := range cs.clusters { + if v := atomic.AddInt32(&ci.refCount, -1); v == 0 { + needUpdate = true + } + } + // We stop the old config selector immediately after sending a new config + // selector; we need another update to delete clusters from the config (if + // we don't have another update pending already). + if needUpdate { + select { + case cs.r.updateCh <- suWithError{emptyUpdate: true}: + default: + } + } +} + +// A global for testing. +var newWRR = wrr.NewRandom + +// newConfigSelector creates the config selector for su; may add entries to +// r.activeClusters for previously-unseen clusters. +func (r *xdsResolver) newConfigSelector(su serviceUpdate) (*configSelector, error) { + cs := &configSelector{ + r: r, + virtualHost: virtualHost{ + httpFilterConfigOverride: su.virtualHost.HTTPFilterConfigOverride, + retryConfig: su.virtualHost.RetryConfig, + }, + routes: make([]route, len(su.virtualHost.Routes)), + clusters: make(map[string]*clusterInfo), + httpFilterConfig: su.ldsConfig.httpFilterConfig, + } + + for i, rt := range su.virtualHost.Routes { + clusters := newWRR() + if rt.ClusterSpecifierPlugin != "" { + clusterName := clusterSpecifierPluginPrefix + rt.ClusterSpecifierPlugin + clusters.Add(&routeCluster{ + name: clusterName, + }, 1) + cs.initializeCluster(clusterName, xdsChildConfig{ + ChildPolicy: balancerConfig(su.clusterSpecifierPlugins[rt.ClusterSpecifierPlugin]), + }) + } else { + for cluster, wc := range rt.WeightedClusters { + clusterName := clusterPrefix + cluster + clusters.Add(&routeCluster{ + name: clusterName, + httpFilterConfigOverride: wc.HTTPFilterConfigOverride, + }, int64(wc.Weight)) + cs.initializeCluster(clusterName, xdsChildConfig{ + ChildPolicy: newBalancerConfig(cdsName, cdsBalancerConfig{Cluster: cluster}), + }) + } + } + cs.routes[i].clusters = clusters + + var err error + cs.routes[i].m, err = resource.RouteToMatcher(rt) + if err != nil { + return nil, err + } + if rt.MaxStreamDuration == nil { + cs.routes[i].maxStreamDuration = su.ldsConfig.maxStreamDuration + } else { + cs.routes[i].maxStreamDuration = *rt.MaxStreamDuration + } + + cs.routes[i].httpFilterConfigOverride = rt.HTTPFilterConfigOverride + cs.routes[i].retryConfig = rt.RetryConfig + cs.routes[i].hashPolicies = rt.HashPolicies + } + + // Account for this config selector's clusters. Do this after no further + // errors may occur. Note: cs.clusters are pointers to entries in + // activeClusters. + for _, ci := range cs.clusters { + atomic.AddInt32(&ci.refCount, 1) + } + + return cs, nil +} + +// initializeCluster initializes entries in cs.clusters map, creating entries in +// r.activeClusters as necessary. Any created entries will have a ref count set +// to zero as their ref count will be incremented by incRefs. +func (cs *configSelector) initializeCluster(clusterName string, cfg xdsChildConfig) { + ci := cs.r.activeClusters[clusterName] + if ci == nil { + ci = &clusterInfo{refCount: 0} + cs.r.activeClusters[clusterName] = ci + } + cs.clusters[clusterName] = ci + cs.clusters[clusterName].cfg = cfg +} + +type clusterInfo struct { + // number of references to this cluster; accessed atomically + refCount int32 + // cfg is the child configuration for this cluster, containing either the + // csp config or the cds cluster config. + cfg xdsChildConfig +} + +type interceptorList struct { + interceptors []iresolver.ClientInterceptor +} + +func (il *interceptorList) NewStream(ctx context.Context, ri iresolver.RPCInfo, done func(), newStream func(ctx context.Context, done func()) (iresolver.ClientStream, error)) (iresolver.ClientStream, error) { + for i := len(il.interceptors) - 1; i >= 0; i-- { + ns := newStream + interceptor := il.interceptors[i] + newStream = func(ctx context.Context, done func()) (iresolver.ClientStream, error) { + return interceptor.NewStream(ctx, ri, done, ns) + } + } + return newStream(ctx, func() {}) +} diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go new file mode 100644 index 0000000000..905aef7390 --- /dev/null +++ b/xds/resolver/watch_service.go @@ -0,0 +1,199 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package resolver + +import ( + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + +// serviceUpdate contains information received from the LDS/RDS responses which +// are of interest to the xds resolver. The RDS request is built by first +// making a LDS to get the RouteConfig name. +type serviceUpdate struct { + // virtualHost contains routes and other configuration to route RPCs. + virtualHost *resource.VirtualHost + // clusterSpecifierPlugins contains the configurations for any cluster + // specifier plugins emitted by the client. + clusterSpecifierPlugins map[string]clusterspecifier.BalancerConfig + // ldsConfig contains configuration that applies to all routes. + ldsConfig ldsConfig +} + +// ldsConfig contains information received from the LDS responses which are of +// interest to the xds resolver. +type ldsConfig struct { + // maxStreamDuration is from the HTTP connection manager's + // common_http_protocol_options field. + maxStreamDuration time.Duration + httpFilterConfig []resource.HTTPFilter +} + +// watchService uses LDS and RDS to discover information about the provided +// serviceName. +// +// Note that during race (e.g. an xDS response is received while the user is +// calling cancel()), there's a small window where the callback can be called +// after the watcher is canceled. The caller needs to handle this case. +func watchService(c client.XDSClient, serviceName string, cb func(serviceUpdate, error), logger *grpclog.PrefixLogger) (cancel func()) { + w := &serviceUpdateWatcher{ + logger: logger, + c: c, + serviceName: serviceName, + serviceCb: cb, + } + w.ldsCancel = c.WatchListener(serviceName, w.handleLDSResp) + + return w.close +} + +// serviceUpdateWatcher handles LDS and RDS response, and calls the service +// callback at the right time. +type serviceUpdateWatcher struct { + logger *grpclog.PrefixLogger + c client.XDSClient + serviceName string + ldsCancel func() + serviceCb func(serviceUpdate, error) + lastUpdate serviceUpdate + + mu sync.Mutex + closed bool + rdsName string + rdsCancel func() +} + +func (w *serviceUpdateWatcher) handleLDSResp(update resource.ListenerUpdate, err error) { + w.logger.Infof("received LDS update: %+v, err: %v", pretty.ToJSON(update), err) + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + if err != nil { + // We check the error type and do different things. For now, the only + // type we check is ResourceNotFound, which indicates the LDS resource + // was removed, and besides sending the error to callback, we also + // cancel the RDS watch. + if resource.ErrType(err) == resource.ErrorTypeResourceNotFound && w.rdsCancel != nil { + w.rdsCancel() + w.rdsName = "" + w.rdsCancel = nil + w.lastUpdate = serviceUpdate{} + } + // The other error cases still return early without canceling the + // existing RDS watch. + w.serviceCb(serviceUpdate{}, err) + return + } + + w.lastUpdate.ldsConfig = ldsConfig{ + maxStreamDuration: update.MaxStreamDuration, + httpFilterConfig: update.HTTPFilters, + } + + if update.InlineRouteConfig != nil { + // If there was an RDS watch, cancel it. + w.rdsName = "" + if w.rdsCancel != nil { + w.rdsCancel() + w.rdsCancel = nil + } + + // Handle the inline RDS update as if it's from an RDS watch. + w.applyRouteConfigUpdate(*update.InlineRouteConfig) + return + } + + // RDS name from update is not an empty string, need RDS to fetch the + // routes. + + if w.rdsName == update.RouteConfigName { + // If the new RouteConfigName is same as the previous, don't cancel and + // restart the RDS watch. + // + // If the route name did change, then we must wait until the first RDS + // update before reporting this LDS config. + if w.lastUpdate.virtualHost != nil { + // We want to send an update with the new fields from the new LDS + // (e.g. max stream duration), and old fields from the the previous + // RDS. + // + // But note that this should only happen when virtual host is set, + // which means an RDS was received. + w.serviceCb(w.lastUpdate, nil) + } + return + } + w.rdsName = update.RouteConfigName + if w.rdsCancel != nil { + w.rdsCancel() + } + w.rdsCancel = w.c.WatchRouteConfig(update.RouteConfigName, w.handleRDSResp) +} + +func (w *serviceUpdateWatcher) applyRouteConfigUpdate(update resource.RouteConfigUpdate) { + matchVh := resource.FindBestMatchingVirtualHost(w.serviceName, update.VirtualHosts) + if matchVh == nil { + // No matching virtual host found. + w.serviceCb(serviceUpdate{}, fmt.Errorf("no matching virtual host found for %q", w.serviceName)) + return + } + + w.lastUpdate.virtualHost = matchVh + w.lastUpdate.clusterSpecifierPlugins = update.ClusterSpecifierPlugins + w.serviceCb(w.lastUpdate, nil) +} + +func (w *serviceUpdateWatcher) handleRDSResp(update resource.RouteConfigUpdate, err error) { + w.logger.Infof("received RDS update: %+v, err: %v", pretty.ToJSON(update), err) + w.mu.Lock() + defer w.mu.Unlock() + if w.closed { + return + } + if w.rdsCancel == nil { + // This mean only the RDS watch is canceled, can happen if the LDS + // resource is removed. + return + } + if err != nil { + w.serviceCb(serviceUpdate{}, err) + return + } + w.applyRouteConfigUpdate(update) +} + +func (w *serviceUpdateWatcher) close() { + w.mu.Lock() + defer w.mu.Unlock() + w.closed = true + w.ldsCancel() + if w.rdsCancel != nil { + w.rdsCancel() + w.rdsCancel = nil + } +} diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go new file mode 100644 index 0000000000..0627826dc2 --- /dev/null +++ b/xds/resolver/xds_resolver.go @@ -0,0 +1,315 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver implements the xds resolver, that does LDS and RDS to find +// the cluster to use. +package resolver + +import ( + "errors" + "fmt" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/resolver" +) + +const xdsScheme = "xds" + +// NewBuilder creates a new xds resolver builder using a specific xds bootstrap +// config, so tests can use multiple xds clients in different ClientConns at +// the same time. +func NewBuilder(config []byte) (resolver.Builder, error) { + return &xdsResolverBuilder{ + newXDSClient: func() (client.XDSClient, error) { + return client.NewClientWithBootstrapContents(config) + }, + }, nil +} + +// For overriding in unittests. +var newXDSClient = func() (client.XDSClient, error) { return client.New() } + +func init() { + resolver.Register(&xdsResolverBuilder{}) +} + +type xdsResolverBuilder struct { + newXDSClient func() (client.XDSClient, error) +} + +// Build helps implement the resolver.Builder interface. +// +// The xds bootstrap process is performed (and a new xds client is built) every +// time an xds resolver is built. +func (b *xdsResolverBuilder) Build(t resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (_ resolver.Resolver, retErr error) { + r := &xdsResolver{ + target: t, + cc: cc, + closed: grpcsync.NewEvent(), + updateCh: make(chan suWithError, 1), + activeClusters: make(map[string]*clusterInfo), + } + defer func() { + if retErr != nil { + r.Close() + } + }() + r.logger = prefixLogger(r) + r.logger.Infof("Creating resolver for target: %+v", t) + + newXDSClient := newXDSClient + if b.newXDSClient != nil { + newXDSClient = b.newXDSClient + } + + client, err := newXDSClient() + if err != nil { + return nil, fmt.Errorf("xds: failed to create xds-client: %v", err) + } + r.client = client + bootstrapConfig := client.BootstrapConfig() + if bootstrapConfig == nil { + return nil, errors.New("bootstrap configuration is empty") + } + + // If xds credentials were specified by the user, but bootstrap configs do + // not contain any certificate provider configuration, it is better to fail + // right now rather than failing when attempting to create certificate + // providers after receiving an CDS response with security configuration. + var creds credentials.TransportCredentials + switch { + case opts.DialCreds != nil: + creds = opts.DialCreds + case opts.CredsBundle != nil: + creds = opts.CredsBundle.TransportCredentials() + } + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + if len(bootstrapConfig.CertProviderConfigs) == 0 { + return nil, errors.New("xds: xdsCreds specified but certificate_providers config missing in bootstrap file") + } + } + + // Find the client listener template to use from the bootstrap config: + // - If authority is not set in the target, use the top level template + // - If authority is set, use the template from the authority map. + template := bootstrapConfig.ClientDefaultListenerResourceNameTemplate + if authority := r.target.URL.Host; authority != "" { + a := bootstrapConfig.Authorities[authority] + if a == nil { + return nil, fmt.Errorf("xds: authority %q is not found in the bootstrap file", authority) + } + if a.ClientListenerResourceNameTemplate != "" { + // This check will never be false, because + // ClientListenerResourceNameTemplate is required to start with + // xdstp://, and has a default value (not an empty string) if unset. + template = a.ClientListenerResourceNameTemplate + } + } + endpoint := r.target.URL.Path + if endpoint == "" { + endpoint = r.target.URL.Opaque + } + endpoint = strings.TrimPrefix(endpoint, "/") + resourceName := bootstrap.PopulateResourceTemplate(template, endpoint) + + // Register a watch on the xdsClient for the user's dial target. + cancelWatch := watchService(r.client, resourceName, r.handleServiceUpdate, r.logger) + r.logger.Infof("Watch started on resource name %v with xds-client %p", r.target.Endpoint, r.client) + r.cancelWatch = func() { + cancelWatch() + r.logger.Infof("Watch cancel on resource name %v with xds-client %p", r.target.Endpoint, r.client) + } + + go r.run() + return r, nil +} + +// Name helps implement the resolver.Builder interface. +func (*xdsResolverBuilder) Scheme() string { + return xdsScheme +} + +// suWithError wraps the ServiceUpdate and error received through a watch API +// callback, so that it can pushed onto the update channel as a single entity. +type suWithError struct { + su serviceUpdate + emptyUpdate bool + err error +} + +// xdsResolver implements the resolver.Resolver interface. +// +// It registers a watcher for ServiceConfig updates with the xdsClient object +// (which performs LDS/RDS queries for the same), and passes the received +// updates to the ClientConn. +type xdsResolver struct { + target resolver.Target + cc resolver.ClientConn + closed *grpcsync.Event + + logger *grpclog.PrefixLogger + + // The underlying xdsClient which performs all xDS requests and responses. + client client.XDSClient + // A channel for the watch API callback to write service updates on to. The + // updates are read by the run goroutine and passed on to the ClientConn. + updateCh chan suWithError + // cancelWatch is the function to cancel the watcher. + cancelWatch func() + + // activeClusters is a map from cluster name to a ref count. Only read or + // written during a service update (synchronous). + activeClusters map[string]*clusterInfo + + curConfigSelector *configSelector +} + +// sendNewServiceConfig prunes active clusters, generates a new service config +// based on the current set of active clusters, and sends an update to the +// channel with that service config and the provided config selector. Returns +// false if an error occurs while generating the service config and the update +// cannot be sent. +func (r *xdsResolver) sendNewServiceConfig(cs *configSelector) bool { + // Delete entries from r.activeClusters with zero references; + // otherwise serviceConfigJSON will generate a config including + // them. + r.pruneActiveClusters() + + if cs == nil && len(r.activeClusters) == 0 { + // There are no clusters and we are sending a failing configSelector. + // Send an empty config, which picks pick-first, with no address, and + // puts the ClientConn into transient failure. + r.cc.UpdateState(resolver.State{ServiceConfig: r.cc.ParseServiceConfig("{}")}) + return true + } + + sc, err := serviceConfigJSON(r.activeClusters) + if err != nil { + // JSON marshal error; should never happen. + r.logger.Errorf("%v", err) + r.cc.ReportError(err) + return false + } + r.logger.Infof("Received update on resource %v from xds-client %p, generated service config: %v", r.target.Endpoint, r.client, pretty.FormatJSON(sc)) + + // Send the update to the ClientConn. + state := iresolver.SetConfigSelector(resolver.State{ + ServiceConfig: r.cc.ParseServiceConfig(string(sc)), + }, cs) + r.cc.UpdateState(client.SetClient(state, r.client)) + return true +} + +// run is a long running goroutine which blocks on receiving service updates +// and passes it on the ClientConn. +func (r *xdsResolver) run() { + for { + select { + case <-r.closed.Done(): + return + case update := <-r.updateCh: + if update.err != nil { + r.logger.Warningf("Watch error on resource %v from xds-client %p, %v", r.target.Endpoint, r.client, update.err) + if resource.ErrType(update.err) == resource.ErrorTypeResourceNotFound { + // If error is resource-not-found, it means the LDS + // resource was removed. Ultimately send an empty service + // config, which picks pick-first, with no address, and + // puts the ClientConn into transient failure. Before we + // can do that, we may need to send a normal service config + // along with an erroring (nil) config selector. + r.sendNewServiceConfig(nil) + // Stop and dereference the active config selector, if one exists. + r.curConfigSelector.stop() + r.curConfigSelector = nil + continue + } + // Send error to ClientConn, and balancers, if error is not + // resource not found. No need to update resolver state if we + // can keep using the old config. + r.cc.ReportError(update.err) + continue + } + if update.emptyUpdate { + r.sendNewServiceConfig(r.curConfigSelector) + continue + } + + // Create the config selector for this update. + cs, err := r.newConfigSelector(update.su) + if err != nil { + r.logger.Warningf("Error parsing update on resource %v from xds-client %p: %v", r.target.Endpoint, r.client, err) + r.cc.ReportError(err) + continue + } + + if !r.sendNewServiceConfig(cs) { + // JSON error creating the service config (unexpected); erase + // this config selector and ignore this update, continuing with + // the previous config selector. + cs.stop() + continue + } + + // Decrement references to the old config selector and assign the + // new one as the current one. + r.curConfigSelector.stop() + r.curConfigSelector = cs + } + } +} + +// handleServiceUpdate is the callback which handles service updates. It writes +// the received update to the update channel, which is picked by the run +// goroutine. +func (r *xdsResolver) handleServiceUpdate(su serviceUpdate, err error) { + if r.closed.HasFired() { + // Do not pass updates to the ClientConn once the resolver is closed. + return + } + // Remove any existing entry in updateCh and replace with the new one. + select { + case <-r.updateCh: + default: + } + r.updateCh <- suWithError{su: su, err: err} +} + +// ResolveNow is a no-op at this point. +func (*xdsResolver) ResolveNow(o resolver.ResolveNowOptions) {} + +// Close closes the resolver, and also closes the underlying client. +func (r *xdsResolver) Close() { + // Note that Close needs to check for nils even if some of them are always + // set in the constructor. This is because the constructor defers Close() in + // error cases, and the fields might not be set when the error happens. + if r.cancelWatch != nil { + r.cancelWatch() + } + if r.client != nil { + r.client.Close() + } + r.closed.Fire() + r.logger.Infof("Shutdown") +} diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go new file mode 100644 index 0000000000..7cd305252a --- /dev/null +++ b/xds/utils/backoff/backoff.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package backoff implement the backoff strategy for gRPC. +// +// This is kept in internal until the gRPC project decides whether or not to +// allow alternative backoff strategies. +package backoff + +import ( + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + grpcbackoff "google.golang.org/grpc/backoff" +) + +// Strategy defines the methodology for backing off after a grpc connection +// failure. +type Strategy interface { + // Backoff returns the amount of time to wait before the next retry given + // the number of consecutive failures. + Backoff(retries int) time.Duration +} + +// DefaultExponential is an exponential backoff implementation using the +// default values for all the configurable knobs defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +var DefaultExponential = Exponential{Config: grpcbackoff.DefaultConfig} + +// Exponential implements exponential backoff algorithm as defined in +// https://github.com/grpc/grpc/blob/master/doc/connection-backoff.md. +type Exponential struct { + // Config contains all options to configure the backoff algorithm. + Config grpcbackoff.Config +} + +// Backoff returns the amount of time to wait before the next retry given the +// number of retries. +func (bc Exponential) Backoff(retries int) time.Duration { + if retries == 0 { + return bc.Config.BaseDelay + } + backoff, max := float64(bc.Config.BaseDelay), float64(bc.Config.MaxDelay) + for backoff < max && retries > 0 { + backoff *= bc.Config.Multiplier + retries-- + } + if backoff > max { + backoff = max + } + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + bc.Config.Jitter*(grpcrand.Float64()*2-1) + if backoff < 0 { + return 0 + } + return time.Duration(backoff) +} diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go new file mode 100644 index 0000000000..950eaaa027 --- /dev/null +++ b/xds/utils/balancer/stub/stub.go @@ -0,0 +1,104 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package stub implements a balancer for testing purposes. +package stub + +import "google.golang.org/grpc/balancer" + +// BalancerFuncs contains all balancer.Balancer functions with a preceding +// *BalancerData parameter for passing additional instance information. Any +// nil functions will never be called. +type BalancerFuncs struct { + // Init is called after ClientConn and BuildOptions are set in + // BalancerData. It may be used to initialize BalancerData.Data. + Init func(*BalancerData) + + UpdateClientConnState func(*BalancerData, balancer.ClientConnState) error + ResolverError func(*BalancerData, error) + UpdateSubConnState func(*BalancerData, balancer.SubConn, balancer.SubConnState) + Close func(*BalancerData) + ExitIdle func(*BalancerData) +} + +// BalancerData contains data relevant to a stub balancer. +type BalancerData struct { + // ClientConn is set by the builder. + ClientConn balancer.ClientConn + // BuildOptions is set by the builder. + BuildOptions balancer.BuildOptions + // Data may be used to store arbitrary user data. + Data interface{} +} + +type bal struct { + bf BalancerFuncs + bd *BalancerData +} + +func (b *bal) UpdateClientConnState(c balancer.ClientConnState) error { + if b.bf.UpdateClientConnState != nil { + return b.bf.UpdateClientConnState(b.bd, c) + } + return nil +} + +func (b *bal) ResolverError(e error) { + if b.bf.ResolverError != nil { + b.bf.ResolverError(b.bd, e) + } +} + +func (b *bal) UpdateSubConnState(sc balancer.SubConn, scs balancer.SubConnState) { + if b.bf.UpdateSubConnState != nil { + b.bf.UpdateSubConnState(b.bd, sc, scs) + } +} + +func (b *bal) Close() { + if b.bf.Close != nil { + b.bf.Close(b.bd) + } +} + +func (b *bal) ExitIdle() { + if b.bf.ExitIdle != nil { + b.bf.ExitIdle(b.bd) + } +} + +type bb struct { + name string + bf BalancerFuncs +} + +func (bb bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer { + b := &bal{bf: bb.bf, bd: &BalancerData{ClientConn: cc, BuildOptions: opts}} + if b.bf.Init != nil { + b.bf.Init(b.bd) + } + return b +} + +func (bb bb) Name() string { return bb.name } + +// Register registers a stub balancer builder which will call the provided +// functions. The name used should be unique. +func Register(name string, bf BalancerFuncs) { + balancer.Register(bb{name: name, bf: bf}) +} diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go new file mode 100644 index 0000000000..477ac7a0f4 --- /dev/null +++ b/xds/utils/balancergroup/balancergroup.go @@ -0,0 +1,531 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package balancergroup implements a utility struct to bind multiple balancers +// into one balancer. +package balancergroup + +import ( + "fmt" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" +) + +// subBalancerWrapper is used to keep the configurations that will be used to start +// the underlying balancer. It can be called to start/stop the underlying +// balancer. +// +// When the config changes, it will pass the update to the underlying balancer +// if it exists. +// +// TODO: move to a separate file? +type subBalancerWrapper struct { + // subBalancerWrapper is passed to the sub-balancer as a ClientConn + // wrapper, only to keep the state and picker. When sub-balancer is + // restarted while in cache, the picker needs to be resent. + // + // It also contains the sub-balancer ID, so the parent balancer group can + // keep track of SubConn/pickers and the sub-balancers they belong to. Some + // of the actions are forwarded to the parent ClientConn with no change. + // Some are forward to balancer group with the sub-balancer ID. + balancer.ClientConn + id string + group *BalancerGroup + + mu sync.Mutex + state balancer.State + + // The static part of sub-balancer. Keeps balancerBuilders and addresses. + // To be used when restarting sub-balancer. + builder balancer.Builder + // Options to be passed to sub-balancer at the time of creation. + buildOpts balancer.BuildOptions + // ccState is a cache of the addresses/balancer config, so when the balancer + // is restarted after close, it will get the previous update. It's a pointer + // and is set to nil at init, so when the balancer is built for the first + // time (not a restart), it won't receive an empty update. Note that this + // isn't reset to nil when the underlying balancer is closed. + ccState *balancer.ClientConnState + // The dynamic part of sub-balancer. Only used when balancer group is + // started. Gets cleared when sub-balancer is closed. + balancer balancer.Balancer +} + +// UpdateState overrides balancer.ClientConn, to keep state and picker. +func (sbc *subBalancerWrapper) UpdateState(state balancer.State) { + sbc.mu.Lock() + sbc.state = state + sbc.group.updateBalancerState(sbc.id, state) + sbc.mu.Unlock() +} + +// NewSubConn overrides balancer.ClientConn, so balancer group can keep track of +// the relation between subconns and sub-balancers. +func (sbc *subBalancerWrapper) NewSubConn(addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + return sbc.group.newSubConn(sbc, addrs, opts) +} + +func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() { + sbc.mu.Lock() + if sbc.state.Picker != nil { + sbc.group.updateBalancerState(sbc.id, sbc.state) + } + sbc.mu.Unlock() +} + +func (sbc *subBalancerWrapper) startBalancer() { + b := sbc.builder.Build(sbc, sbc.buildOpts) + sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name()) + sbc.balancer = b + if sbc.ccState != nil { + b.UpdateClientConnState(*sbc.ccState) + } +} + +// exitIdle invokes the sub-balancer's ExitIdle method. Returns a boolean +// indicating whether or not the operation was completed. +func (sbc *subBalancerWrapper) exitIdle() (complete bool) { + b := sbc.balancer + if b == nil { + return true + } + if ei, ok := b.(balancer.ExitIdler); ok { + ei.ExitIdle() + return true + } + return false +} + +func (sbc *subBalancerWrapper) updateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + b := sbc.balancer + if b == nil { + // This sub-balancer was closed. This can happen when EDS removes a + // locality. The balancer for this locality was already closed, and the + // SubConns are being deleted. But SubConn state change can still + // happen. + return + } + b.UpdateSubConnState(sc, state) +} + +func (sbc *subBalancerWrapper) updateClientConnState(s balancer.ClientConnState) error { + sbc.ccState = &s + b := sbc.balancer + if b == nil { + // This sub-balancer was closed. This should never happen because + // sub-balancers are closed when the locality is removed from EDS, or + // the balancer group is closed. There should be no further address + // updates when either of this happened. + // + // This will be a common case with priority support, because a + // sub-balancer (and the whole balancer group) could be closed because + // it's the lower priority, but it can still get address updates. + return nil + } + return b.UpdateClientConnState(s) +} + +func (sbc *subBalancerWrapper) resolverError(err error) { + b := sbc.balancer + if b == nil { + // This sub-balancer was closed. This should never happen because + // sub-balancers are closed when the locality is removed from EDS, or + // the balancer group is closed. There should be no further address + // updates when either of this happened. + // + // This will be a common case with priority support, because a + // sub-balancer (and the whole balancer group) could be closed because + // it's the lower priority, but it can still get address updates. + return + } + b.ResolverError(err) +} + +func (sbc *subBalancerWrapper) stopBalancer() { + sbc.balancer.Close() + sbc.balancer = nil +} + +// BalancerGroup takes a list of balancers, and make them into one balancer. +// +// Note that this struct doesn't implement balancer.Balancer, because it's not +// intended to be used directly as a balancer. It's expected to be used as a +// sub-balancer manager by a high level balancer. +// +// Updates from ClientConn are forwarded to sub-balancers +// - service config update +// - address update +// - subConn state change +// - find the corresponding balancer and forward +// +// Actions from sub-balances are forwarded to parent ClientConn +// - new/remove SubConn +// - picker update and health states change +// - sub-pickers are sent to an aggregator provided by the parent, which +// will group them into a group-picker. The aggregated connectivity state is +// also handled by the aggregator. +// - resolveNow +// +// Sub-balancers are only built when the balancer group is started. If the +// balancer group is closed, the sub-balancers are also closed. And it's +// guaranteed that no updates will be sent to parent ClientConn from a closed +// balancer group. +type BalancerGroup struct { + cc balancer.ClientConn + buildOpts balancer.BuildOptions + logger *grpclog.PrefixLogger + + // stateAggregator is where the state/picker updates will be sent to. It's + // provided by the parent balancer, to build a picker with all the + // sub-pickers. + stateAggregator BalancerStateAggregator + + // outgoingMu guards all operations in the direction: + // ClientConn-->Sub-balancer. Including start, stop, resolver updates and + // SubConn state changes. + // + // The corresponding boolean outgoingStarted is used to stop further updates + // to sub-balancers after they are closed. + outgoingMu sync.Mutex + outgoingStarted bool + idToBalancerConfig map[string]*subBalancerWrapper + // Cache for sub-balancers when they are removed. + balancerCache *cache.TimeoutCache + + // incomingMu is to make sure this balancer group doesn't send updates to cc + // after it's closed. + // + // We don't share the mutex to avoid deadlocks (e.g. a call to sub-balancer + // may call back to balancer group inline. It causes deaclock if they + // require the same mutex). + // + // We should never need to hold multiple locks at the same time in this + // struct. The case where two locks are held can only happen when the + // underlying balancer calls back into balancer group inline. So there's an + // implicit lock acquisition order that outgoingMu is locked before + // incomingMu. + + // incomingMu guards all operations in the direction: + // Sub-balancer-->ClientConn. Including NewSubConn, RemoveSubConn. It also + // guards the map from SubConn to balancer ID, so updateSubConnState needs + // to hold it shortly to find the sub-balancer to forward the update. + // + // UpdateState is called by the balancer state aggretator, and it will + // decide when and whether to call. + // + // The corresponding boolean incomingStarted is used to stop further updates + // from sub-balancers after they are closed. + incomingMu sync.Mutex + incomingStarted bool // This boolean only guards calls back to ClientConn. + scToSubBalancer map[balancer.SubConn]*subBalancerWrapper +} + +// DefaultSubBalancerCloseTimeout is defined as a variable instead of const for +// testing. +// +// TODO: make it a parameter for New(). +var DefaultSubBalancerCloseTimeout = 15 * time.Minute + +// New creates a new BalancerGroup. Note that the BalancerGroup +// needs to be started to work. +func New(cc balancer.ClientConn, bOpts balancer.BuildOptions, stateAggregator BalancerStateAggregator, logger *grpclog.PrefixLogger) *BalancerGroup { + return &BalancerGroup{ + cc: cc, + buildOpts: bOpts, + logger: logger, + stateAggregator: stateAggregator, + + idToBalancerConfig: make(map[string]*subBalancerWrapper), + balancerCache: cache.NewTimeoutCache(DefaultSubBalancerCloseTimeout), + scToSubBalancer: make(map[balancer.SubConn]*subBalancerWrapper), + } +} + +// Start starts the balancer group, including building all the sub-balancers, +// and send the existing addresses to them. +// +// A BalancerGroup can be closed and started later. When a BalancerGroup is +// closed, it can still receive address updates, which will be applied when +// restarted. +func (bg *BalancerGroup) Start() { + bg.incomingMu.Lock() + bg.incomingStarted = true + bg.incomingMu.Unlock() + + bg.outgoingMu.Lock() + if bg.outgoingStarted { + bg.outgoingMu.Unlock() + return + } + + for _, config := range bg.idToBalancerConfig { + config.startBalancer() + } + bg.outgoingStarted = true + bg.outgoingMu.Unlock() +} + +// Add adds a balancer built by builder to the group, with given id. +func (bg *BalancerGroup) Add(id string, builder balancer.Builder) { + // Store data in static map, and then check to see if bg is started. + bg.outgoingMu.Lock() + var sbc *subBalancerWrapper + // If outgoingStarted is true, search in the cache. Otherwise, cache is + // guaranteed to be empty, searching is unnecessary. + if bg.outgoingStarted { + if old, ok := bg.balancerCache.Remove(id); ok { + sbc, _ = old.(*subBalancerWrapper) + if sbc != nil && sbc.builder != builder { + // If the sub-balancer in cache was built with a different + // balancer builder, don't use it, cleanup this old-balancer, + // and behave as sub-balancer is not found in cache. + // + // NOTE that this will also drop the cached addresses for this + // sub-balancer, which seems to be reasonable. + sbc.stopBalancer() + // cleanupSubConns must be done before the new balancer starts, + // otherwise new SubConns created by the new balancer might be + // removed by mistake. + bg.cleanupSubConns(sbc) + sbc = nil + } + } + } + if sbc == nil { + sbc = &subBalancerWrapper{ + ClientConn: bg.cc, + id: id, + group: bg, + builder: builder, + buildOpts: bg.buildOpts, + } + if bg.outgoingStarted { + // Only start the balancer if bg is started. Otherwise, we only keep the + // static data. + sbc.startBalancer() + } + } else { + // When brining back a sub-balancer from cache, re-send the cached + // picker and state. + sbc.updateBalancerStateWithCachedPicker() + } + bg.idToBalancerConfig[id] = sbc + bg.outgoingMu.Unlock() +} + +// Remove removes the balancer with id from the group. +// +// But doesn't close the balancer. The balancer is kept in a cache, and will be +// closed after timeout. Cleanup work (closing sub-balancer and removing +// subconns) will be done after timeout. +func (bg *BalancerGroup) Remove(id string) { + bg.outgoingMu.Lock() + if sbToRemove, ok := bg.idToBalancerConfig[id]; ok { + if bg.outgoingStarted { + bg.balancerCache.Add(id, sbToRemove, func() { + // After timeout, when sub-balancer is removed from cache, need + // to close the underlying sub-balancer, and remove all its + // subconns. + bg.outgoingMu.Lock() + if bg.outgoingStarted { + sbToRemove.stopBalancer() + } + bg.outgoingMu.Unlock() + bg.cleanupSubConns(sbToRemove) + }) + } + delete(bg.idToBalancerConfig, id) + } else { + bg.logger.Infof("balancer group: trying to remove a non-existing locality from balancer group: %v", id) + } + bg.outgoingMu.Unlock() +} + +// bg.remove(id) doesn't do cleanup for the sub-balancer. This function does +// cleanup after the timeout. +func (bg *BalancerGroup) cleanupSubConns(config *subBalancerWrapper) { + bg.incomingMu.Lock() + // Remove SubConns. This is only done after the balancer is + // actually closed. + // + // NOTE: if NewSubConn is called by this (closed) balancer later, the + // SubConn will be leaked. This shouldn't happen if the balancer + // implementation is correct. To make sure this never happens, we need to + // add another layer (balancer manager) between balancer group and the + // sub-balancers. + for sc, b := range bg.scToSubBalancer { + if b == config { + bg.cc.RemoveSubConn(sc) + delete(bg.scToSubBalancer, sc) + } + } + bg.incomingMu.Unlock() +} + +// connect attempts to connect to all subConns belonging to sb. +func (bg *BalancerGroup) connect(sb *subBalancerWrapper) { + bg.incomingMu.Lock() + for sc, b := range bg.scToSubBalancer { + if b == sb { + sc.Connect() + } + } + bg.incomingMu.Unlock() +} + +// Following are actions from the parent grpc.ClientConn, forward to sub-balancers. + +// UpdateSubConnState handles the state for the subconn. It finds the +// corresponding balancer and forwards the update. +func (bg *BalancerGroup) UpdateSubConnState(sc balancer.SubConn, state balancer.SubConnState) { + bg.incomingMu.Lock() + config, ok := bg.scToSubBalancer[sc] + if !ok { + bg.incomingMu.Unlock() + return + } + if state.ConnectivityState == connectivity.Shutdown { + // Only delete sc from the map when state changed to Shutdown. + delete(bg.scToSubBalancer, sc) + } + bg.incomingMu.Unlock() + + bg.outgoingMu.Lock() + config.updateSubConnState(sc, state) + bg.outgoingMu.Unlock() +} + +// UpdateClientConnState handles ClientState (including balancer config and +// addresses) from resolver. It finds the balancer and forwards the update. +func (bg *BalancerGroup) UpdateClientConnState(id string, s balancer.ClientConnState) error { + bg.outgoingMu.Lock() + defer bg.outgoingMu.Unlock() + if config, ok := bg.idToBalancerConfig[id]; ok { + return config.updateClientConnState(s) + } + return nil +} + +// ResolverError forwards resolver errors to all sub-balancers. +func (bg *BalancerGroup) ResolverError(err error) { + bg.outgoingMu.Lock() + for _, config := range bg.idToBalancerConfig { + config.resolverError(err) + } + bg.outgoingMu.Unlock() +} + +// Following are actions from sub-balancers, forward to ClientConn. + +// newSubConn: forward to ClientConn, and also create a map from sc to balancer, +// so state update will find the right balancer. +// +// One note about removing SubConn: only forward to ClientConn, but not delete +// from map. Delete sc from the map only when state changes to Shutdown. Since +// it's just forwarding the action, there's no need for a removeSubConn() +// wrapper function. +func (bg *BalancerGroup) newSubConn(config *subBalancerWrapper, addrs []resolver.Address, opts balancer.NewSubConnOptions) (balancer.SubConn, error) { + // NOTE: if balancer with id was already removed, this should also return + // error. But since we call balancer.stopBalancer when removing the balancer, this + // shouldn't happen. + bg.incomingMu.Lock() + if !bg.incomingStarted { + bg.incomingMu.Unlock() + return nil, fmt.Errorf("NewSubConn is called after balancer group is closed") + } + sc, err := bg.cc.NewSubConn(addrs, opts) + if err != nil { + bg.incomingMu.Unlock() + return nil, err + } + bg.scToSubBalancer[sc] = config + bg.incomingMu.Unlock() + return sc, nil +} + +// updateBalancerState: forward the new state to balancer state aggregator. The +// aggregator will create an aggregated picker and an aggregated connectivity +// state, then forward to ClientConn. +func (bg *BalancerGroup) updateBalancerState(id string, state balancer.State) { + bg.logger.Infof("Balancer state update from locality %v, new state: %+v", id, state) + + // Send new state to the aggregator, without holding the incomingMu. + // incomingMu is to protect all calls to the parent ClientConn, this update + // doesn't necessary trigger a call to ClientConn, and should already be + // protected by aggregator's mutex if necessary. + if bg.stateAggregator != nil { + bg.stateAggregator.UpdateState(id, state) + } +} + +// Close closes the balancer. It stops sub-balancers, and removes the subconns. +// The BalancerGroup can be restarted later. +func (bg *BalancerGroup) Close() { + bg.incomingMu.Lock() + if bg.incomingStarted { + bg.incomingStarted = false + // Also remove all SubConns. + for sc := range bg.scToSubBalancer { + bg.cc.RemoveSubConn(sc) + delete(bg.scToSubBalancer, sc) + } + } + bg.incomingMu.Unlock() + + // Clear(true) runs clear function to close sub-balancers in cache. It + // must be called out of outgoing mutex. + bg.balancerCache.Clear(true) + + bg.outgoingMu.Lock() + if bg.outgoingStarted { + bg.outgoingStarted = false + for _, config := range bg.idToBalancerConfig { + config.stopBalancer() + } + } + bg.outgoingMu.Unlock() +} + +// ExitIdle should be invoked when the parent LB policy's ExitIdle is invoked. +// It will trigger this on all sub-balancers, or reconnect their subconns if +// not supported. +func (bg *BalancerGroup) ExitIdle() { + bg.outgoingMu.Lock() + for _, config := range bg.idToBalancerConfig { + if !config.exitIdle() { + bg.connect(config) + } + } + bg.outgoingMu.Unlock() +} + +// ExitIdleOne instructs the sub-balancer `id` to exit IDLE state, if +// appropriate and possible. +func (bg *BalancerGroup) ExitIdleOne(id string) { + bg.outgoingMu.Lock() + if config := bg.idToBalancerConfig[id]; config != nil { + if !config.exitIdle() { + bg.connect(config) + } + } + bg.outgoingMu.Unlock() +} diff --git a/xds/utils/balancergroup/balancerstateaggregator.go b/xds/utils/balancergroup/balancerstateaggregator.go new file mode 100644 index 0000000000..1163943850 --- /dev/null +++ b/xds/utils/balancergroup/balancerstateaggregator.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package balancergroup + +import ( + "google.golang.org/grpc/balancer" +) + +// BalancerStateAggregator aggregates sub-picker and connectivity states into a +// state. +// +// It takes care of merging sub-picker into one picker. The picking config is +// passed directly from the the parent to the aggregator implementation (instead +// via balancer group). +type BalancerStateAggregator interface { + // UpdateState updates the state of the id. + // + // It's up to the implementation whether this will trigger an update to the + // parent ClientConn. + UpdateState(id string, state balancer.State) +} diff --git a/xds/utils/balancerload/load.go b/xds/utils/balancerload/load.go new file mode 100644 index 0000000000..3a905d9665 --- /dev/null +++ b/xds/utils/balancerload/load.go @@ -0,0 +1,46 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package balancerload defines APIs to parse server loads in trailers. The +// parsed loads are sent to balancers in DoneInfo. +package balancerload + +import ( + "google.golang.org/grpc/metadata" +) + +// Parser converts loads from metadata into a concrete type. +type Parser interface { + // Parse parses loads from metadata. + Parse(md metadata.MD) interface{} +} + +var parser Parser + +// SetParser sets the load parser. +// +// Not mutex-protected, should be called before any gRPC functions. +func SetParser(lr Parser) { + parser = lr +} + +// Parse calls parser.Read(). +func Parse(md metadata.MD) interface{} { + if parser == nil { + return nil + } + return parser.Parse(md) +} diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go new file mode 100644 index 0000000000..9f6a0c1200 --- /dev/null +++ b/xds/utils/buffer/unbounded.go @@ -0,0 +1,85 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package buffer provides an implementation of an unbounded buffer. +package buffer + +import "sync" + +// Unbounded is an implementation of an unbounded buffer which does not use +// extra goroutines. This is typically used for passing updates from one entity +// to another within gRPC. +// +// All methods on this type are thread-safe and don't block on anything except +// the underlying mutex used for synchronization. +// +// Unbounded supports values of any type to be stored in it by using a channel +// of `interface{}`. This means that a call to Put() incurs an extra memory +// allocation, and also that users need a type assertion while reading. For +// performance critical code paths, using Unbounded is strongly discouraged and +// defining a new type specific implementation of this buffer is preferred. See +// internal/transport/transport.go for an example of this. +type Unbounded struct { + c chan interface{} + mu sync.Mutex + backlog []interface{} +} + +// NewUnbounded returns a new instance of Unbounded. +func NewUnbounded() *Unbounded { + return &Unbounded{c: make(chan interface{}, 1)} +} + +// Put adds t to the unbounded buffer. +func (b *Unbounded) Put(t interface{}) { + b.mu.Lock() + if len(b.backlog) == 0 { + select { + case b.c <- t: + b.mu.Unlock() + return + default: + } + } + b.backlog = append(b.backlog, t) + b.mu.Unlock() +} + +// Load sends the earliest buffered data, if any, onto the read channel +// returned by Get(). Users are expected to call this every time they read a +// value from the read channel. +func (b *Unbounded) Load() { + b.mu.Lock() + if len(b.backlog) > 0 { + select { + case b.c <- b.backlog[0]: + b.backlog[0] = nil + b.backlog = b.backlog[1:] + default: + } + } + b.mu.Unlock() +} + +// Get returns a read channel on which values added to the buffer, via Put(), +// are sent on. +// +// Upon reading a value from this channel, users are expected to call Load() to +// send the next buffered value onto the channel if there is any. +func (b *Unbounded) Get() <-chan interface{} { + return b.c +} diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go new file mode 100644 index 0000000000..eaca00b818 --- /dev/null +++ b/xds/utils/credentials/xds/handshake_info.go @@ -0,0 +1,318 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package xds contains non-user facing functionality of the xds credentials. +package xds + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "strings" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/resolver" +) + +func init() { + //internal.GetXDSHandshakeInfoForTesting = GetHandshakeInfo +} + +// handshakeAttrKey is the type used as the key to store HandshakeInfo in +// the Attributes field of resolver.Address. +type handshakeAttrKey struct{} + +// Equal reports whether the handshake info structs are identical (have the +// same pointer). This is sufficient as all subconns from one CDS balancer use +// the same one. +func (hi *HandshakeInfo) Equal(o interface{}) bool { + oh, ok := o.(*HandshakeInfo) + return ok && oh == hi +} + +// SetHandshakeInfo returns a copy of addr in which the Attributes field is +// updated with hInfo. +func SetHandshakeInfo(addr resolver.Address, hInfo *HandshakeInfo) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeAttrKey{}, hInfo) + return addr +} + +// GetHandshakeInfo returns a pointer to the HandshakeInfo stored in attr. +func GetHandshakeInfo(attr *attributes.Attributes) *HandshakeInfo { + v := attr.Value(handshakeAttrKey{}) + hi, _ := v.(*HandshakeInfo) + return hi +} + +// HandshakeInfo wraps all the security configuration required by client and +// server handshake methods in xds credentials. The xDS implementation will be +// responsible for populating these fields. +// +// Safe for concurrent access. +type HandshakeInfo struct { + mu sync.Mutex + rootProvider certprovider.Provider + identityProvider certprovider.Provider + sanMatchers []matcher.StringMatcher // Only on the client side. + requireClientCert bool // Only on server side. +} + +// SetRootCertProvider updates the root certificate provider. +func (hi *HandshakeInfo) SetRootCertProvider(root certprovider.Provider) { + hi.mu.Lock() + hi.rootProvider = root + hi.mu.Unlock() +} + +// SetIdentityCertProvider updates the identity certificate provider. +func (hi *HandshakeInfo) SetIdentityCertProvider(identity certprovider.Provider) { + hi.mu.Lock() + hi.identityProvider = identity + hi.mu.Unlock() +} + +// SetSANMatchers updates the list of SAN matchers. +func (hi *HandshakeInfo) SetSANMatchers(sanMatchers []matcher.StringMatcher) { + hi.mu.Lock() + hi.sanMatchers = sanMatchers + hi.mu.Unlock() +} + +// SetRequireClientCert updates whether a client cert is required during the +// ServerHandshake(). A value of true indicates that we are performing mTLS. +func (hi *HandshakeInfo) SetRequireClientCert(require bool) { + hi.mu.Lock() + hi.requireClientCert = require + hi.mu.Unlock() +} + +// UseFallbackCreds returns true when fallback credentials are to be used based +// on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) UseFallbackCreds() bool { + if hi == nil { + return true + } + + hi.mu.Lock() + defer hi.mu.Unlock() + return hi.identityProvider == nil && hi.rootProvider == nil +} + +// GetSANMatchersForTesting returns the SAN matchers stored in HandshakeInfo. +// To be used only for testing purposes. +func (hi *HandshakeInfo) GetSANMatchersForTesting() []matcher.StringMatcher { + hi.mu.Lock() + defer hi.mu.Unlock() + return append([]matcher.StringMatcher{}, hi.sanMatchers...) +} + +// ClientSideTLSConfig constructs a tls.Config to be used in a client-side +// handshake based on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) ClientSideTLSConfig(ctx context.Context) (*tls.Config, error) { + hi.mu.Lock() + // On the client side, rootProvider is mandatory. IdentityProvider is + // optional based on whether the client is doing TLS or mTLS. + if hi.rootProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch trusted roots is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + // Since the call to KeyMaterial() can block, we read the providers under + // the lock but call the actual function after releasing the lock. + rootProv, idProv := hi.rootProvider, hi.identityProvider + hi.mu.Unlock() + + // InsecureSkipVerify needs to be set to true because we need to perform + // custom verification to check the SAN on the received certificate. + // Currently the Go stdlib does complete verification of the cert (which + // includes hostname verification) or none. We are forced to go with the + // latter and perform the normal cert validation ourselves. + cfg := &tls.Config{ + InsecureSkipVerify: true, + NextProtos: []string{"h2"}, + } + + km, err := rootProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + cfg.RootCAs = km.Roots + + if idProv != nil { + km, err := idProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + } + return cfg, nil +} + +// ServerSideTLSConfig constructs a tls.Config to be used in a server-side +// handshake based on the contents of the HandshakeInfo. +func (hi *HandshakeInfo) ServerSideTLSConfig(ctx context.Context) (*tls.Config, error) { + cfg := &tls.Config{ + ClientAuth: tls.NoClientCert, + NextProtos: []string{"h2"}, + } + hi.mu.Lock() + // On the server side, identityProvider is mandatory. RootProvider is + // optional based on whether the server is doing TLS or mTLS. + if hi.identityProvider == nil { + return nil, errors.New("xds: CertificateProvider to fetch identity certificate is missing, cannot perform TLS handshake. Please check configuration on the management server") + } + // Since the call to KeyMaterial() can block, we read the providers under + // the lock but call the actual function after releasing the lock. + rootProv, idProv := hi.rootProvider, hi.identityProvider + if hi.requireClientCert { + cfg.ClientAuth = tls.RequireAndVerifyClientCert + } + hi.mu.Unlock() + + // identityProvider is mandatory on the server side. + km, err := idProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching identity certificates from CertificateProvider failed: %v", err) + } + cfg.Certificates = km.Certs + + if rootProv != nil { + km, err := rootProv.KeyMaterial(ctx) + if err != nil { + return nil, fmt.Errorf("xds: fetching trusted roots from CertificateProvider failed: %v", err) + } + cfg.ClientCAs = km.Roots + } + return cfg, nil +} + +// MatchingSANExists returns true if the SANs contained in cert match the +// criteria enforced by the list of SAN matchers in HandshakeInfo. +// +// If the list of SAN matchers in the HandshakeInfo is empty, this function +// returns true for all input certificates. +func (hi *HandshakeInfo) MatchingSANExists(cert *x509.Certificate) bool { + hi.mu.Lock() + defer hi.mu.Unlock() + if len(hi.sanMatchers) == 0 { + return true + } + + // SANs can be specified in any of these four fields on the parsed cert. + for _, san := range cert.DNSNames { + if hi.matchSAN(san, true) { + return true + } + } + for _, san := range cert.EmailAddresses { + if hi.matchSAN(san, false) { + return true + } + } + for _, san := range cert.IPAddresses { + if hi.matchSAN(san.String(), false) { + return true + } + } + for _, san := range cert.URIs { + if hi.matchSAN(san.String(), false) { + return true + } + } + return false +} + +// Caller must hold mu. +func (hi *HandshakeInfo) matchSAN(san string, isDNS bool) bool { + for _, matcher := range hi.sanMatchers { + if em := matcher.ExactMatch(); em != "" && isDNS { + // This is a special case which is documented in the xDS protos. + // If the DNS SAN is a wildcard entry, and the match criteria is + // `exact`, then we need to perform DNS wildcard matching + // instead of regular string comparison. + if dnsMatch(em, san) { + return true + } + continue + } + if matcher.Match(san) { + return true + } + } + return false +} + +// dnsMatch implements a DNS wildcard matching algorithm based on RFC2828 and +// grpc-java's implementation in `OkHostnameVerifier` class. +// +// NOTE: Here the `host` argument is the one from the set of string matchers in +// the xDS proto and the `san` argument is a DNS SAN from the certificate, and +// this is the one which can potentially contain a wildcard pattern. +func dnsMatch(host, san string) bool { + // Add trailing "." and turn them into absolute domain names. + if !strings.HasSuffix(host, ".") { + host += "." + } + if !strings.HasSuffix(san, ".") { + san += "." + } + // Domain names are case-insensitive. + host = strings.ToLower(host) + san = strings.ToLower(san) + + // If san does not contain a wildcard, do exact match. + if !strings.Contains(san, "*") { + return host == san + } + + // Wildcard dns matching rules + // - '*' is only permitted in the left-most label and must be the only + // character in that label. For example, *.example.com is permitted, while + // *a.example.com, a*.example.com, a*b.example.com, a.*.example.com are + // not permitted. + // - '*' matches a single domain name component. For example, *.example.com + // matches test.example.com but does not match sub.test.example.com. + // - Wildcard patterns for single-label domain names are not permitted. + if san == "*." || !strings.HasPrefix(san, "*.") || strings.Contains(san[1:], "*") { + return false + } + // Optimization: at this point, we know that the san contains a '*' and + // is the first domain component of san. So, the host name must be at + // least as long as the san to be able to match. + if len(host) < len(san) { + return false + } + // Hostname must end with the non-wildcard portion of san. + if !strings.HasSuffix(host, san[1:]) { + return false + } + // At this point we know that the hostName and san share the same suffix + // (the non-wildcard portion of san). Now, we just need to make sure + // that the '*' does not match across domain components. + hostPrefix := strings.TrimSuffix(host, san[1:]) + return !strings.Contains(hostPrefix, ".") +} + +// NewHandshakeInfo returns a new instance of HandshakeInfo with the given root +// and identity certificate providers. +func NewHandshakeInfo(root, identity certprovider.Provider) *HandshakeInfo { + return &HandshakeInfo{rootProvider: root, identityProvider: identity} +} diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go new file mode 100644 index 0000000000..9e0683bb3e --- /dev/null +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -0,0 +1,304 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "crypto/x509" + "net" + "net/url" + "regexp" + "testing" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + +func TestDNSMatch(t *testing.T) { + tests := []struct { + desc string + host string + pattern string + wantMatch bool + }{ + { + desc: "invalid wildcard 1", + host: "aa.example.com", + pattern: "*a.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 2", + host: "aa.example.com", + pattern: "a*.example.com", + wantMatch: false, + }, + { + desc: "invalid wildcard 3", + host: "abc.example.com", + pattern: "a*c.example.com", + wantMatch: false, + }, + { + desc: "wildcard in one of the middle components", + host: "abc.test.example.com", + pattern: "abc.*.example.com", + wantMatch: false, + }, + { + desc: "single component wildcard", + host: "a.example.com", + pattern: "*", + wantMatch: false, + }, + { + desc: "short host name", + host: "a.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "suffix mismatch", + host: "a.notexample.com", + pattern: "*.example.com", + wantMatch: false, + }, + { + desc: "wildcard match across components", + host: "sub.test.example.com", + pattern: "*.example.com.", + wantMatch: false, + }, + { + desc: "host doesn't end in period", + host: "test.example.com", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "pattern doesn't end in period", + host: "test.example.com.", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "case insensitive", + host: "TEST.EXAMPLE.COM.", + pattern: "test.example.com.", + wantMatch: true, + }, + { + desc: "simple match", + host: "test.example.com", + pattern: "test.example.com", + wantMatch: true, + }, + { + desc: "good wildcard", + host: "a.example.com", + pattern: "*.example.com", + wantMatch: true, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatch := dnsMatch(test.host, test.pattern) + if gotMatch != test.wantMatch { + t.Fatalf("dnsMatch(%s, %s) = %v, want %v", test.host, test.pattern, gotMatch, test.wantMatch) + } + }) + } +} + +func TestMatchingSANExists_FailureCases(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"foo.bar.example.com", "bar.baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")}, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "exact match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("abcd.test.com"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("http://golang"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("HTTP://GOLANG.ORG"), nil, nil, nil, nil, false), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("i-aint-the-one"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("FOO.BAR"), nil, nil, nil, false), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP("i-aint-the-one"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP("1::68"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP(".COM"), nil, nil, false), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.examples\.com`), false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("i-aint-the-one"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:db8:1:1::68"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, false), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil) + hi.SetSANMatchers(test.sanMatchers) + + if hi.MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v succeeded when expected to fail", inputCert, test.sanMatchers) + } + }) + } +} + +func TestMatchingSANExists_Success(t *testing.T) { + url1, err := url.Parse("http://golang.org") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + url2, err := url.Parse("https://github.com/grpc/grpc-go") + if err != nil { + t.Fatalf("url.Parse() failed: %v", err) + } + inputCert := &x509.Certificate{ + DNSNames: []string{"baz.test.com", "*.example.com"}, + EmailAddresses: []string{"foobar@example.com", "barbaz@test.com"}, + IPAddresses: []net.IP{net.ParseIP("192.0.0.1"), net.ParseIP("2001:db8::68")}, + URIs: []*url.URL{url1, url2}, + } + + tests := []struct { + desc string + sanMatchers []matcher.StringMatcher + }{ + { + desc: "no san matchers", + }, + { + desc: "exact match dns wildcard", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(newStringP("abc.example.com"), nil, nil, nil, nil, false), + }, + }, + { + desc: "exact match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("FOOBAR@EXAMPLE.COM"), nil, nil, nil, nil, true), + }, + }, + { + desc: "prefix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP(".co.in"), nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("192.168.1.1"), nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, newStringP("baz.test"), nil, nil, nil, false), + }, + }, + { + desc: "prefix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, newStringP("BAZ.test"), nil, nil, nil, true), + }, + }, + { + desc: "suffix match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + matcher.StringMatcherForTesting(nil, nil, newStringP("192.168.1.1"), nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, newStringP("@test.com"), nil, nil, false), + }, + }, + { + desc: "suffix match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, newStringP("@test.COM"), nil, nil, true), + }, + }, + { + desc: "regex match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("https://github.com/grpc/grpc-java"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`192\.[0-9]{1,3}\.1\.1`), false), + matcher.StringMatcherForTesting(nil, nil, nil, nil, regexp.MustCompile(`.*\.test\.com`), false), + }, + }, + { + desc: "contains match", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(newStringP("https://github.com/grpc/grpc-java"), nil, nil, nil, nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("2001:68::db8"), nil, false), + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("192.0.0"), nil, false), + }, + }, + { + desc: "contains match ignore case", + sanMatchers: []matcher.StringMatcher{ + matcher.StringMatcherForTesting(nil, nil, nil, newStringP("GRPC"), nil, true), + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + hi := NewHandshakeInfo(nil, nil) + hi.SetSANMatchers(test.sanMatchers) + + if !hi.MatchingSANExists(inputCert) { + t.Fatalf("hi.MatchingSANExists(%+v) with SAN matchers +%v failed when expected to succeed", inputCert, test.sanMatchers) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/xds/utils/envconfig/envconfig.go b/xds/utils/envconfig/envconfig.go new file mode 100644 index 0000000000..6f02725431 --- /dev/null +++ b/xds/utils/envconfig/envconfig.go @@ -0,0 +1,35 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package envconfig contains grpc settings configured by environment variables. +package envconfig + +import ( + "os" + "strings" +) + +const ( + prefix = "GRPC_GO_" + txtErrIgnoreStr = prefix + "IGNORE_TXT_ERRORS" +) + +var ( + // TXTErrIgnore is set if TXT errors should be ignored ("GRPC_GO_IGNORE_TXT_ERRORS" is not "false"). + TXTErrIgnore = !strings.EqualFold(os.Getenv(txtErrIgnoreStr), "false") +) diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go new file mode 100644 index 0000000000..9bad03cec6 --- /dev/null +++ b/xds/utils/envconfig/xds.go @@ -0,0 +1,97 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package envconfig + +import ( + "os" + "strings" +) + +const ( + // XDSBootstrapFileNameEnv is the env variable to set bootstrap file name. + // Do not use this and read from env directly. Its value is read and kept in + // variable BootstrapFileName. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileNameEnv = "GRPC_XDS_BOOTSTRAP" + // XDSBootstrapFileContentEnv is the env variable to set bootstrapp file + // content. Do not use this and read from env directly. Its value is read + // and kept in variable BootstrapFileName. + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContentEnv = "GRPC_XDS_BOOTSTRAP_CONFIG" + + ringHashSupportEnv = "GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH" + clientSideSecuritySupportEnv = "GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT" + aggregateAndDNSSupportEnv = "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER" + rbacSupportEnv = "GRPC_XDS_EXPERIMENTAL_RBAC" + federationEnv = "GRPC_EXPERIMENTAL_XDS_FEDERATION" + rlsInXDSEnv = "GRPC_EXPERIMENTAL_XDS_RLS_LB" + + c2pResolverTestOnlyTrafficDirectorURIEnv = "GRPC_TEST_ONLY_GOOGLE_C2P_RESOLVER_TRAFFIC_DIRECTOR_URI" +) + +var ( + // XDSBootstrapFileName holds the name of the file which contains xDS + // bootstrap configuration. Users can specify the location of the bootstrap + // file by setting the environment variable "GRPC_XDS_BOOTSTRAP". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileName = os.Getenv(XDSBootstrapFileNameEnv) + // XDSBootstrapFileContent holds the content of the xDS bootstrap + // configuration. Users can specify the bootstrap config by setting the + // environment variable "GRPC_XDS_BOOTSTRAP_CONFIG". + // + // When both bootstrap FileName and FileContent are set, FileName is used. + XDSBootstrapFileContent = os.Getenv(XDSBootstrapFileContentEnv) + // XDSRingHash indicates whether ring hash support is enabled, which can be + // disabled by setting the environment variable + // "GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH" to "false". + XDSRingHash = !strings.EqualFold(os.Getenv(ringHashSupportEnv), "false") + // XDSClientSideSecurity is used to control processing of security + // configuration on the client-side. + // + // Note that there is no env var protection for the server-side because we + // have a brand new API on the server-side and users explicitly need to use + // the new API to get security integration on the server. + XDSClientSideSecurity = !strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "false") + // XDSAggregateAndDNS indicates whether processing of aggregated cluster + // and DNS cluster is enabled, which can be enabled by setting the + // environment variable + // "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER" to + // "true". + XDSAggregateAndDNS = strings.EqualFold(os.Getenv(aggregateAndDNSSupportEnv), "true") + + // XDSRBAC indicates whether xDS configured RBAC HTTP Filter is enabled, + // which can be disabled by setting the environment variable + // "GRPC_XDS_EXPERIMENTAL_RBAC" to "false". + XDSRBAC = !strings.EqualFold(os.Getenv(rbacSupportEnv), "false") + + // XDSFederation indicates whether federation support is enabled. + XDSFederation = strings.EqualFold(os.Getenv(federationEnv), "true") + + // XDSRLS indicates whether processing of Cluster Specifier plugins and + // support for the RLS CLuster Specifier is enabled, which can be enabled by + // setting the environment variable "GRPC_EXPERIMENTAL_XDS_RLS_LB" to + // "true". + XDSRLS = strings.EqualFold(os.Getenv(rlsInXDSEnv), "true") + + // C2PResolverTestOnlyTrafficDirectorURI is the TD URI for testing. + C2PResolverTestOnlyTrafficDirectorURI = os.Getenv(c2pResolverTestOnlyTrafficDirectorURIEnv) +) diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go new file mode 100644 index 0000000000..30a3b4258f --- /dev/null +++ b/xds/utils/grpclog/grpclog.go @@ -0,0 +1,126 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpclog (internal) defines depth logging for grpc. +package grpclog + +import ( + "os" +) + +// Logger is the logger used for the non-depth log functions. +var Logger LoggerV2 + +// DepthLogger is the logger used for the depth log functions. +var DepthLogger DepthLoggerV2 + +// InfoDepth logs to the INFO log at the specified depth. +func InfoDepth(depth int, args ...interface{}) { + if DepthLogger != nil { + DepthLogger.InfoDepth(depth, args...) + } else { + Logger.Infoln(args...) + } +} + +// WarningDepth logs to the WARNING log at the specified depth. +func WarningDepth(depth int, args ...interface{}) { + if DepthLogger != nil { + DepthLogger.WarningDepth(depth, args...) + } else { + Logger.Warningln(args...) + } +} + +// ErrorDepth logs to the ERROR log at the specified depth. +func ErrorDepth(depth int, args ...interface{}) { + if DepthLogger != nil { + DepthLogger.ErrorDepth(depth, args...) + } else { + Logger.Errorln(args...) + } +} + +// FatalDepth logs to the FATAL log at the specified depth. +func FatalDepth(depth int, args ...interface{}) { + if DepthLogger != nil { + DepthLogger.FatalDepth(depth, args...) + } else { + Logger.Fatalln(args...) + } + os.Exit(1) +} + +// LoggerV2 does underlying logging work for grpclog. +// This is a copy of the LoggerV2 defined in the external grpclog package. It +// is defined here to avoid a circular dependency. +type LoggerV2 interface { + // Info logs to INFO log. Arguments are handled in the manner of fmt.Print. + Info(args ...interface{}) + // Infoln logs to INFO log. Arguments are handled in the manner of fmt.Println. + Infoln(args ...interface{}) + // Infof logs to INFO log. Arguments are handled in the manner of fmt.Printf. + Infof(format string, args ...interface{}) + // Warning logs to WARNING log. Arguments are handled in the manner of fmt.Print. + Warning(args ...interface{}) + // Warningln logs to WARNING log. Arguments are handled in the manner of fmt.Println. + Warningln(args ...interface{}) + // Warningf logs to WARNING log. Arguments are handled in the manner of fmt.Printf. + Warningf(format string, args ...interface{}) + // Error logs to ERROR log. Arguments are handled in the manner of fmt.Print. + Error(args ...interface{}) + // Errorln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + Errorln(args ...interface{}) + // Errorf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + Errorf(format string, args ...interface{}) + // Fatal logs to ERROR log. Arguments are handled in the manner of fmt.Print. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatal(args ...interface{}) + // Fatalln logs to ERROR log. Arguments are handled in the manner of fmt.Println. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalln(args ...interface{}) + // Fatalf logs to ERROR log. Arguments are handled in the manner of fmt.Printf. + // gRPC ensures that all Fatal logs will exit with os.Exit(1). + // Implementations may also call os.Exit() with a non-zero exit code. + Fatalf(format string, args ...interface{}) + // V reports whether verbosity level l is at least the requested verbose level. + V(l int) bool +} + +// DepthLoggerV2 logs at a specified call frame. If a LoggerV2 also implements +// DepthLoggerV2, the below functions will be called with the appropriate stack +// depth set for trivial functions the logger may ignore. +// This is a copy of the DepthLoggerV2 defined in the external grpclog package. +// It is defined here to avoid a circular dependency. +// +// Experimental +// +// Notice: This type is EXPERIMENTAL and may be changed or removed in a +// later release. +type DepthLoggerV2 interface { + // InfoDepth logs to INFO log at the specified depth. Arguments are handled in the manner of fmt.Println. + InfoDepth(depth int, args ...interface{}) + // WarningDepth logs to WARNING log at the specified depth. Arguments are handled in the manner of fmt.Println. + WarningDepth(depth int, args ...interface{}) + // ErrorDepth logs to ERROR log at the specified depth. Arguments are handled in the manner of fmt.Println. + ErrorDepth(depth int, args ...interface{}) + // FatalDepth logs to FATAL log at the specified depth. Arguments are handled in the manner of fmt.Println. + FatalDepth(depth int, args ...interface{}) +} diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go new file mode 100644 index 0000000000..82af70e96f --- /dev/null +++ b/xds/utils/grpclog/prefixLogger.go @@ -0,0 +1,81 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpclog + +import ( + "fmt" +) + +// PrefixLogger does logging with a prefix. +// +// Logging method on a nil logs without any prefix. +type PrefixLogger struct { + logger DepthLoggerV2 + prefix string +} + +// Infof does info logging. +func (pl *PrefixLogger) Infof(format string, args ...interface{}) { + if pl != nil { + // Handle nil, so the tests can pass in a nil logger. + format = pl.prefix + format + pl.logger.InfoDepth(1, fmt.Sprintf(format, args...)) + return + } + InfoDepth(1, fmt.Sprintf(format, args...)) +} + +// Warningf does warning logging. +func (pl *PrefixLogger) Warningf(format string, args ...interface{}) { + if pl != nil { + format = pl.prefix + format + pl.logger.WarningDepth(1, fmt.Sprintf(format, args...)) + return + } + WarningDepth(1, fmt.Sprintf(format, args...)) +} + +// Errorf does error logging. +func (pl *PrefixLogger) Errorf(format string, args ...interface{}) { + if pl != nil { + format = pl.prefix + format + pl.logger.ErrorDepth(1, fmt.Sprintf(format, args...)) + return + } + ErrorDepth(1, fmt.Sprintf(format, args...)) +} + +// Debugf does info logging at verbose level 2. +func (pl *PrefixLogger) Debugf(format string, args ...interface{}) { + if !Logger.V(2) { + return + } + if pl != nil { + // Handle nil, so the tests can pass in a nil logger. + format = pl.prefix + format + pl.logger.InfoDepth(1, fmt.Sprintf(format, args...)) + return + } + InfoDepth(1, fmt.Sprintf(format, args...)) +} + +// NewPrefixLogger creates a prefix logger with the given prefix. +func NewPrefixLogger(logger DepthLoggerV2, prefix string) *PrefixLogger { + return &PrefixLogger{logger: logger, prefix: prefix} +} diff --git a/xds/utils/grpcrand/grpcrand.go b/xds/utils/grpcrand/grpcrand.go new file mode 100644 index 0000000000..740f83c2b7 --- /dev/null +++ b/xds/utils/grpcrand/grpcrand.go @@ -0,0 +1,67 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcrand implements math/rand functions in a concurrent-safe way +// with a global random source, independent of math/rand's global source. +package grpcrand + +import ( + "math/rand" + "sync" + "time" +) + +var ( + r = rand.New(rand.NewSource(time.Now().UnixNano())) + mu sync.Mutex +) + +// Int implements rand.Int on the grpcrand global source. +func Int() int { + mu.Lock() + defer mu.Unlock() + return r.Int() +} + +// Int63n implements rand.Int63n on the grpcrand global source. +func Int63n(n int64) int64 { + mu.Lock() + defer mu.Unlock() + return r.Int63n(n) +} + +// Intn implements rand.Intn on the grpcrand global source. +func Intn(n int) int { + mu.Lock() + defer mu.Unlock() + return r.Intn(n) +} + +// Float64 implements rand.Float64 on the grpcrand global source. +func Float64() float64 { + mu.Lock() + defer mu.Unlock() + return r.Float64() +} + +// Uint64 implements rand.Uint64 on the grpcrand global source. +func Uint64() uint64 { + mu.Lock() + defer mu.Unlock() + return r.Uint64() +} diff --git a/xds/utils/grpcsync/event.go b/xds/utils/grpcsync/event.go new file mode 100644 index 0000000000..fbe697c376 --- /dev/null +++ b/xds/utils/grpcsync/event.go @@ -0,0 +1,61 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcsync implements additional synchronization primitives built upon +// the sync package. +package grpcsync + +import ( + "sync" + "sync/atomic" +) + +// Event represents a one-time event that may occur in the future. +type Event struct { + fired int32 + c chan struct{} + o sync.Once +} + +// Fire causes e to complete. It is safe to call multiple times, and +// concurrently. It returns true iff this call to Fire caused the signaling +// channel returned by Done to close. +func (e *Event) Fire() bool { + ret := false + e.o.Do(func() { + atomic.StoreInt32(&e.fired, 1) + close(e.c) + ret = true + }) + return ret +} + +// Done returns a channel that will be closed when Fire is called. +func (e *Event) Done() <-chan struct{} { + return e.c +} + +// HasFired returns true if Fire has been called. +func (e *Event) HasFired() bool { + return atomic.LoadInt32(&e.fired) == 1 +} + +// NewEvent returns a new, ready-to-use Event. +func NewEvent() *Event { + return &Event{c: make(chan struct{})} +} diff --git a/xds/utils/grpcutil/encode_duration.go b/xds/utils/grpcutil/encode_duration.go new file mode 100644 index 0000000000..b25b0baec3 --- /dev/null +++ b/xds/utils/grpcutil/encode_duration.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "strconv" + "time" +) + +const maxTimeoutValue int64 = 100000000 - 1 + +// div does integer division and round-up the result. Note that this is +// equivalent to (d+r-1)/r but has less chance to overflow. +func div(d, r time.Duration) int64 { + if d%r > 0 { + return int64(d/r + 1) + } + return int64(d / r) +} + +// EncodeDuration encodes the duration to the format grpc-timeout header +// accepts. +// +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests +func EncodeDuration(t time.Duration) string { + // TODO: This is simplistic and not bandwidth efficient. Improve it. + if t <= 0 { + return "0n" + } + if d := div(t, time.Nanosecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "n" + } + if d := div(t, time.Microsecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "u" + } + if d := div(t, time.Millisecond); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "m" + } + if d := div(t, time.Second); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "S" + } + if d := div(t, time.Minute); d <= maxTimeoutValue { + return strconv.FormatInt(d, 10) + "M" + } + // Note that maxTimeoutValue * time.Hour > MaxInt64. + return strconv.FormatInt(div(t, time.Hour), 10) + "H" +} diff --git a/xds/utils/grpcutil/encode_duration_test.go b/xds/utils/grpcutil/encode_duration_test.go new file mode 100644 index 0000000000..eea49e2e77 --- /dev/null +++ b/xds/utils/grpcutil/encode_duration_test.go @@ -0,0 +1,51 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" + "time" +) + +func TestEncodeDuration(t *testing.T) { + for _, test := range []struct { + in string + out string + }{ + {"12345678ns", "12345678n"}, + {"123456789ns", "123457u"}, + {"12345678us", "12345678u"}, + {"123456789us", "123457m"}, + {"12345678ms", "12345678m"}, + {"123456789ms", "123457S"}, + {"12345678s", "12345678S"}, + {"123456789s", "2057614M"}, + {"12345678m", "12345678M"}, + {"123456789m", "2057614H"}, + } { + d, err := time.ParseDuration(test.in) + if err != nil { + t.Fatalf("failed to parse duration string %s: %v", test.in, err) + } + out := EncodeDuration(d) + if out != test.out { + t.Fatalf("timeoutEncode(%s) = %s, want %s", test.in, out, test.out) + } + } +} diff --git a/xds/utils/grpcutil/grpcutil.go b/xds/utils/grpcutil/grpcutil.go new file mode 100644 index 0000000000..e2f948e8f4 --- /dev/null +++ b/xds/utils/grpcutil/grpcutil.go @@ -0,0 +1,20 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package grpcutil provides utility functions used across the gRPC codebase. +package grpcutil diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go new file mode 100644 index 0000000000..6f22bd8911 --- /dev/null +++ b/xds/utils/grpcutil/metadata.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +type mdExtraKey struct{} + +// WithExtraMetadata creates a new context with incoming md attached. +func WithExtraMetadata(ctx context.Context, md metadata.MD) context.Context { + return context.WithValue(ctx, mdExtraKey{}, md) +} + +// ExtraMetadata returns the incoming metadata in ctx if it exists. The +// returned MD should not be modified. Writing to it may cause races. +// Modification should be made to copies of the returned MD. +func ExtraMetadata(ctx context.Context) (md metadata.MD, ok bool) { + md, ok = ctx.Value(mdExtraKey{}).(metadata.MD) + return +} diff --git a/xds/utils/grpcutil/method.go b/xds/utils/grpcutil/method.go new file mode 100644 index 0000000000..4e7475060c --- /dev/null +++ b/xds/utils/grpcutil/method.go @@ -0,0 +1,84 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "errors" + "strings" +) + +// ParseMethod splits service and method from the input. It expects format +// "/service/method". +// +func ParseMethod(methodName string) (service, method string, _ error) { + if !strings.HasPrefix(methodName, "/") { + return "", "", errors.New("invalid method name: should start with /") + } + methodName = methodName[1:] + + pos := strings.LastIndex(methodName, "/") + if pos < 0 { + return "", "", errors.New("invalid method name: suffix /method is missing") + } + return methodName[:pos], methodName[pos+1:], nil +} + +const baseContentType = "application/grpc" + +// ContentSubtype returns the content-subtype for the given content-type. The +// given content-type must be a valid content-type that starts with +// "application/grpc". A content-subtype will follow "application/grpc" after a +// "+" or ";". See +// https://github.com/grpc/grpc/blob/master/doc/PROTOCOL-HTTP2.md#requests for +// more details. +// +// If contentType is not a valid content-type for gRPC, the boolean +// will be false, otherwise true. If content-type == "application/grpc", +// "application/grpc+", or "application/grpc;", the boolean will be true, +// but no content-subtype will be returned. +// +// contentType is assumed to be lowercase already. +func ContentSubtype(contentType string) (string, bool) { + if contentType == baseContentType { + return "", true + } + if !strings.HasPrefix(contentType, baseContentType) { + return "", false + } + // guaranteed since != baseContentType and has baseContentType prefix + switch contentType[len(baseContentType)] { + case '+', ';': + // this will return true for "application/grpc+" or "application/grpc;" + // which the previous validContentType function tested to be valid, so we + // just say that no content-subtype is specified in this case + return contentType[len(baseContentType)+1:], true + default: + return "", false + } +} + +// ContentType builds full content type with the given sub-type. +// +// contentSubtype is assumed to be lowercase +func ContentType(contentSubtype string) string { + if contentSubtype == "" { + return baseContentType + } + return baseContentType + "+" + contentSubtype +} diff --git a/xds/utils/grpcutil/method_test.go b/xds/utils/grpcutil/method_test.go new file mode 100644 index 0000000000..36c786cffb --- /dev/null +++ b/xds/utils/grpcutil/method_test.go @@ -0,0 +1,69 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "testing" +) + +func TestParseMethod(t *testing.T) { + testCases := []struct { + methodName string + wantService string + wantMethod string + wantError bool + }{ + {methodName: "/s/m", wantService: "s", wantMethod: "m", wantError: false}, + {methodName: "/p.s/m", wantService: "p.s", wantMethod: "m", wantError: false}, + {methodName: "/p/s/m", wantService: "p/s", wantMethod: "m", wantError: false}, + {methodName: "/", wantError: true}, + {methodName: "/sm", wantError: true}, + {methodName: "", wantError: true}, + {methodName: "sm", wantError: true}, + } + for _, tc := range testCases { + s, m, err := ParseMethod(tc.methodName) + if (err != nil) != tc.wantError || s != tc.wantService || m != tc.wantMethod { + t.Errorf("ParseMethod(%s) = (%s, %s, %v), want (%s, %s, %v)", tc.methodName, s, m, err, tc.wantService, tc.wantMethod, tc.wantError) + } + } +} + +func TestContentSubtype(t *testing.T) { + tests := []struct { + contentType string + want string + wantValid bool + }{ + {"application/grpc", "", true}, + {"application/grpc+", "", true}, + {"application/grpc+blah", "blah", true}, + {"application/grpc;", "", true}, + {"application/grpc;blah", "blah", true}, + {"application/grpcd", "", false}, + {"application/grpd", "", false}, + {"application/grp", "", false}, + } + for _, tt := range tests { + got, gotValid := ContentSubtype(tt.contentType) + if got != tt.want || gotValid != tt.wantValid { + t.Errorf("contentSubtype(%q) = (%v, %v); want (%v, %v)", tt.contentType, got, gotValid, tt.want, tt.wantValid) + } + } +} diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go new file mode 100644 index 0000000000..7a092b2b80 --- /dev/null +++ b/xds/utils/grpcutil/regex.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import "regexp" + +// FullMatchWithRegex returns whether the full text matches the regex provided. +func FullMatchWithRegex(re *regexp.Regexp, text string) bool { + if len(text) == 0 { + return re.MatchString(text) + } + re.Longest() + rem := re.FindString(text) + return len(rem) == len(text) +} diff --git a/xds/utils/grpcutil/regex_test.go b/xds/utils/grpcutil/regex_test.go new file mode 100644 index 0000000000..4c12804fed --- /dev/null +++ b/xds/utils/grpcutil/regex_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package grpcutil + +import ( + "regexp" + "testing" +) + +func TestFullMatchWithRegex(t *testing.T) { + tests := []struct { + name string + regexStr string + string string + want bool + }{ + { + name: "not match because only partial", + regexStr: "^a+$", + string: "ab", + want: false, + }, + { + name: "match because fully match", + regexStr: "^a+$", + string: "aa", + want: true, + }, + { + name: "longest", + regexStr: "a(|b)", + string: "ab", + want: true, + }, + { + name: "match all", + regexStr: ".*", + string: "", + want: true, + }, + { + name: "matches non-empty strings", + regexStr: ".+", + string: "", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := regexp.MustCompile(tt.regexStr) + if got := FullMatchWithRegex(hrm, tt.string); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/xds/utils/hierarchy/hierarchy.go b/xds/utils/hierarchy/hierarchy.go new file mode 100644 index 0000000000..341d3405dc --- /dev/null +++ b/xds/utils/hierarchy/hierarchy.go @@ -0,0 +1,109 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package hierarchy contains functions to set and get hierarchy string from +// addresses. +// +// This package is experimental. +package hierarchy + +import ( + "google.golang.org/grpc/resolver" +) + +type pathKeyType string + +const pathKey = pathKeyType("grpc.internal.address.hierarchical_path") + +type pathValue []string + +func (p pathValue) Equal(o interface{}) bool { + op, ok := o.(pathValue) + if !ok { + return false + } + if len(op) != len(p) { + return false + } + for i, v := range p { + if v != op[i] { + return false + } + } + return true +} + +// Get returns the hierarchical path of addr. +func Get(addr resolver.Address) []string { + attrs := addr.BalancerAttributes + if attrs == nil { + return nil + } + path, _ := attrs.Value(pathKey).(pathValue) + return ([]string)(path) +} + +// Set overrides the hierarchical path in addr with path. +func Set(addr resolver.Address, path []string) resolver.Address { + addr.BalancerAttributes = addr.BalancerAttributes.WithValue(pathKey, pathValue(path)) + return addr +} + +// Group splits a slice of addresses into groups based on +// the first hierarchy path. The first hierarchy path will be removed from the +// result. +// +// Input: +// [ +// {addr0, path: [p0, wt0]} +// {addr1, path: [p0, wt1]} +// {addr2, path: [p1, wt2]} +// {addr3, path: [p1, wt3]} +// ] +// +// Addresses will be split into p0/p1, and the p0/p1 will be removed from the +// path. +// +// Output: +// { +// p0: [ +// {addr0, path: [wt0]}, +// {addr1, path: [wt1]}, +// ], +// p1: [ +// {addr2, path: [wt2]}, +// {addr3, path: [wt3]}, +// ], +// } +// +// If hierarchical path is not set, or has no path in it, the address is +// dropped. +func Group(addrs []resolver.Address) map[string][]resolver.Address { + ret := make(map[string][]resolver.Address) + for _, addr := range addrs { + oldPath := Get(addr) + if len(oldPath) == 0 { + continue + } + curPath := oldPath[0] + newPath := oldPath[1:] + newAddr := Set(addr, newPath) + ret[curPath] = append(ret[curPath], newAddr) + } + return ret +} diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go new file mode 100644 index 0000000000..1043d5f81d --- /dev/null +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -0,0 +1,197 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package hierarchy + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" +) + +func TestGet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + want []string + }{ + { + name: "not set", + addr: resolver.Address{}, + want: nil, + }, + { + name: "set", + addr: resolver.Address{ + BalancerAttributes: attributes.New(pathKey, pathValue{"a", "b"}), + }, + want: []string{"a", "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Get(tt.addr); !cmp.Equal(got, tt.want) { + t.Errorf("Get() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestSet(t *testing.T) { + tests := []struct { + name string + addr resolver.Address + path []string + }{ + { + name: "before is not set", + addr: resolver.Address{}, + path: []string{"a", "b"}, + }, + { + name: "before is set", + addr: resolver.Address{ + BalancerAttributes: attributes.New(pathKey, pathValue{"before", "a", "b"}), + }, + path: []string{"a", "b"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + newAddr := Set(tt.addr, tt.path) + newPath := Get(newAddr) + if !cmp.Equal(newPath, tt.path) { + t.Errorf("path after Set() = %v, want %v", newPath, tt.path) + } + }) + } +} + +func TestGroup(t *testing.T) { + tests := []struct { + name string + addrs []resolver.Address + want map[string][]resolver.Address + }{ + { + name: "all with hierarchy", + addrs: []resolver.Address{ + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, pathValue{"b"})}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, pathValue{"b"})}, + }, + want: map[string][]resolver.Address{ + "a": { + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + }, + "b": { + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + }, + }, + }, + { + // Addresses without hierarchy are ignored. + name: "without hierarchy", + addrs: []resolver.Address{ + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: nil}, + {Addr: "b1", BalancerAttributes: nil}, + }, + want: map[string][]resolver.Address{ + "a": { + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + }, + }, + }, + { + // If hierarchy is set to a wrong type (which should never happen), + // the address is ignored. + name: "wrong type", + addrs: []resolver.Address{ + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{"a"})}, + {Addr: "b0", BalancerAttributes: attributes.New(pathKey, "b")}, + {Addr: "b1", BalancerAttributes: attributes.New(pathKey, 314)}, + }, + want: map[string][]resolver.Address{ + "a": { + {Addr: "a0", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + {Addr: "a1", BalancerAttributes: attributes.New(pathKey, pathValue{})}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := Group(tt.addrs); !cmp.Equal(got, tt.want, cmp.AllowUnexported(attributes.Attributes{})) { + t.Errorf("Group() = %v, want %v", got, tt.want) + t.Errorf("diff: %v", cmp.Diff(got, tt.want, cmp.AllowUnexported(attributes.Attributes{}))) + } + }) + } +} + +func TestGroupE2E(t *testing.T) { + hierarchy := map[string]map[string][]string{ + "p0": { + "wt0": {"addr0", "addr1"}, + "wt1": {"addr2", "addr3"}, + }, + "p1": { + "wt10": {"addr10", "addr11"}, + "wt11": {"addr12", "addr13"}, + }, + } + + var addrsWithHierarchy []resolver.Address + for p, wts := range hierarchy { + path1 := pathValue{p} + for wt, addrs := range wts { + path2 := append(pathValue(nil), path1...) + path2 = append(path2, wt) + for _, addr := range addrs { + a := resolver.Address{ + Addr: addr, + BalancerAttributes: attributes.New(pathKey, path2), + } + addrsWithHierarchy = append(addrsWithHierarchy, a) + } + } + } + + gotHierarchy := make(map[string]map[string][]string) + for p1, wts := range Group(addrsWithHierarchy) { + gotHierarchy[p1] = make(map[string][]string) + for p2, addrs := range Group(wts) { + for _, addr := range addrs { + gotHierarchy[p1][p2] = append(gotHierarchy[p1][p2], addr.Addr) + } + } + } + + if !cmp.Equal(gotHierarchy, hierarchy) { + t.Errorf("diff: %v", cmp.Diff(gotHierarchy, hierarchy)) + } +} diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go new file mode 100644 index 0000000000..7d9bc0bd76 --- /dev/null +++ b/xds/utils/matcher/matcher_header.go @@ -0,0 +1,241 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "fmt" + "google.golang.org/grpc/metadata" + "regexp" + "strconv" + "strings" +) + +// HeaderMatcher is an interface for header matchers. These are +// documented in (EnvoyProxy link here?). These matchers will match on different +// aspects of HTTP header name/value pairs. +type HeaderMatcher interface { + Match(metadata.MD) bool + String() string +} + +// mdValuesFromOutgoingCtx retrieves metadata from context. If there are +// multiple values, the values are concatenated with "," (comma and no space). +// +// All header matchers only match against the comma-concatenated string. +func mdValuesFromOutgoingCtx(md metadata.MD, key string) (string, bool) { + vs, ok := md[key] + if !ok { + return "", false + } + return strings.Join(vs, ","), true +} + +// HeaderExactMatcher matches on an exact match of the value of the header. +type HeaderExactMatcher struct { + key string + exact string + invert bool +} + +// NewHeaderExactMatcher returns a new HeaderExactMatcher. +func NewHeaderExactMatcher(key, exact string, invert bool) *HeaderExactMatcher { + return &HeaderExactMatcher{key: key, exact: exact, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderExactMatcher. +func (hem *HeaderExactMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hem.key) + if !ok { + return false + } + return (v == hem.exact) != hem.invert +} + +func (hem *HeaderExactMatcher) String() string { + return fmt.Sprintf("headerExact:%v:%v", hem.key, hem.exact) +} + +// HeaderRegexMatcher matches on whether the entire request header value matches +// the regex. +type HeaderRegexMatcher struct { + key string + re *regexp.Regexp + invert bool +} + +// NewHeaderRegexMatcher returns a new HeaderRegexMatcher. +func NewHeaderRegexMatcher(key string, re *regexp.Regexp, invert bool) *HeaderRegexMatcher { + return &HeaderRegexMatcher{key: key, re: re, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRegexMatcher. +func (hrm *HeaderRegexMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hrm.key) + if !ok { + return false + } + return FullMatchWithRegex(hrm.re, v) != hrm.invert +} + +func (hrm *HeaderRegexMatcher) String() string { + return fmt.Sprintf("headerRegex:%v:%v", hrm.key, hrm.re.String()) +} + +// HeaderRangeMatcher matches on whether the request header value is within the +// range. The header value must be an integer in base 10 notation. +type HeaderRangeMatcher struct { + key string + start, end int64 // represents [start, end). + invert bool +} + +// NewHeaderRangeMatcher returns a new HeaderRangeMatcher. +func NewHeaderRangeMatcher(key string, start, end int64, invert bool) *HeaderRangeMatcher { + return &HeaderRangeMatcher{key: key, start: start, end: end, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderRangeMatcher. +func (hrm *HeaderRangeMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hrm.key) + if !ok { + return false + } + if i, err := strconv.ParseInt(v, 10, 64); err == nil && i >= hrm.start && i < hrm.end { + return !hrm.invert + } + return hrm.invert +} + +func (hrm *HeaderRangeMatcher) String() string { + return fmt.Sprintf("headerRange:%v:[%d,%d)", hrm.key, hrm.start, hrm.end) +} + +// HeaderPresentMatcher will match based on whether the header is present in the +// whole request. +type HeaderPresentMatcher struct { + key string + present bool +} + +// NewHeaderPresentMatcher returns a new HeaderPresentMatcher. +func NewHeaderPresentMatcher(key string, present bool, invert bool) *HeaderPresentMatcher { + if invert { + present = !present + } + return &HeaderPresentMatcher{key: key, present: present} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPresentMatcher. +func (hpm *HeaderPresentMatcher) Match(md metadata.MD) bool { + vs, ok := mdValuesFromOutgoingCtx(md, hpm.key) + present := ok && len(vs) > 0 // TODO: Are we sure we need this len(vs) > 0? + return present == hpm.present +} + +func (hpm *HeaderPresentMatcher) String() string { + return fmt.Sprintf("headerPresent:%v:%v", hpm.key, hpm.present) +} + +// HeaderPrefixMatcher matches on whether the prefix of the header value matches +// the prefix passed into this struct. +type HeaderPrefixMatcher struct { + key string + prefix string + invert bool +} + +// NewHeaderPrefixMatcher returns a new HeaderPrefixMatcher. +func NewHeaderPrefixMatcher(key string, prefix string, invert bool) *HeaderPrefixMatcher { + return &HeaderPrefixMatcher{key: key, prefix: prefix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderPrefixMatcher. +func (hpm *HeaderPrefixMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hpm.key) + if !ok { + return false + } + return strings.HasPrefix(v, hpm.prefix) != hpm.invert +} + +func (hpm *HeaderPrefixMatcher) String() string { + return fmt.Sprintf("headerPrefix:%v:%v", hpm.key, hpm.prefix) +} + +// HeaderSuffixMatcher matches on whether the suffix of the header value matches +// the suffix passed into this struct. +type HeaderSuffixMatcher struct { + key string + suffix string + invert bool +} + +// NewHeaderSuffixMatcher returns a new HeaderSuffixMatcher. +func NewHeaderSuffixMatcher(key string, suffix string, invert bool) *HeaderSuffixMatcher { + return &HeaderSuffixMatcher{key: key, suffix: suffix, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderSuffixMatcher. +func (hsm *HeaderSuffixMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hsm.key) + if !ok { + return false + } + return strings.HasSuffix(v, hsm.suffix) != hsm.invert +} + +func (hsm *HeaderSuffixMatcher) String() string { + return fmt.Sprintf("headerSuffix:%v:%v", hsm.key, hsm.suffix) +} + +// HeaderContainsMatcher matches on whether the header value contains the +// value passed into this struct. +type HeaderContainsMatcher struct { + key string + contains string + invert bool +} + +// NewHeaderContainsMatcher returns a new HeaderContainsMatcher. key is the HTTP +// Header key to match on, and contains is the value that the header should +// should contain for a successful match. An empty contains string does not +// work, use HeaderPresentMatcher in that case. +func NewHeaderContainsMatcher(key string, contains string, invert bool) *HeaderContainsMatcher { + return &HeaderContainsMatcher{key: key, contains: contains, invert: invert} +} + +// Match returns whether the passed in HTTP Headers match according to the +// HeaderContainsMatcher. +func (hcm *HeaderContainsMatcher) Match(md metadata.MD) bool { + v, ok := mdValuesFromOutgoingCtx(md, hcm.key) + if !ok { + return false + } + return strings.Contains(v, hcm.contains) != hcm.invert +} + +func (hcm *HeaderContainsMatcher) String() string { + return fmt.Sprintf("headerContains:%v%v", hcm.key, hcm.contains) +} diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go new file mode 100644 index 0000000000..f567f31982 --- /dev/null +++ b/xds/utils/matcher/matcher_header_test.go @@ -0,0 +1,469 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" + + "google.golang.org/grpc/metadata" +) + +func TestHeaderExactMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, exact string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "two value one match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "abc", "th", "tv"), + // Doesn't match comma-concatenated string. + want: false, + }, + { + name: "two value match concatenated", + key: "th", + exact: "abc,tv", + md: metadata.Pairs("th", "abc", "th", "tv"), + want: true, + }, + { + name: "not match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + exact: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + exact: "tv", + md: metadata.Pairs("th", "tvv"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hem := NewHeaderExactMatcher(tt.key, tt.exact, tt.invert) + if got := hem.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderRegexMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, regexStr string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "tttvv"), + want: true, + }, + { + name: "two value one match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc", "th", "tttvv"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + regexStr: "^[abc]*,t+v*$", + md: metadata.Pairs("th", "abc", "th", "tttvv"), + want: true, + }, + { + name: "no match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "no match because only part of value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "ab"), + want: false, + }, + { + name: "match because full value matches with regex", + key: "header", + regexStr: "^a+$", + md: metadata.Pairs("header", "aa"), + want: true, + }, + { + name: "invert header not present", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "tttvv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + regexStr: "^t+v*$", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := NewHeaderRegexMatcher(tt.key, regexp.MustCompile(tt.regexStr), tt.invert) + if got := hrm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderRangeMatcherMatch(t *testing.T) { + tests := []struct { + name string + key string + start, end int64 + md metadata.MD + want bool + invert bool + }{ + { + name: "match", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "5"), + want: true, + }, + { + name: "equal to start", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "1"), + want: true, + }, + { + name: "equal to end", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "10"), + want: false, + }, + { + name: "negative", + key: "th", + start: -10, end: 10, + md: metadata.Pairs("th", "-5"), + want: true, + }, + { + name: "invert header not present", + key: "th", + start: 1, end: 10, + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + start: 1, end: 10, + md: metadata.Pairs("th", "5"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + start: 1, end: 9, + md: metadata.Pairs("th", "10"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := NewHeaderRangeMatcher(tt.key, tt.start, tt.end, tt.invert) + if got := hrm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderPresentMatcherMatch(t *testing.T) { + tests := []struct { + name string + key string + present bool + md metadata.MD + want bool + invert bool + }{ + { + name: "want present is present", + key: "th", + present: true, + md: metadata.Pairs("th", "tv"), + want: true, + }, + { + name: "want present not present", + key: "th", + present: true, + md: metadata.Pairs("abc", "tv"), + want: false, + }, + { + name: "want not present is present", + key: "th", + present: false, + md: metadata.Pairs("th", "tv"), + want: false, + }, + { + name: "want not present is not present", + key: "th", + present: false, + md: metadata.Pairs("abc", "tv"), + want: true, + }, + { + name: "invert header not present", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, + { + name: "invert header match", + key: "th", + present: true, + md: metadata.Pairs("th", "tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + present: true, + md: metadata.Pairs(":method", "GET"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hpm := NewHeaderPresentMatcher(tt.key, tt.present, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderPrefixMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, prefix string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123"), + want: true, + }, + { + name: "two value one match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc", "th", "tv123"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123", "th", "abc"), + want: true, + }, + { + name: "not match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + prefix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "tv123"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + prefix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hpm := NewHeaderPrefixMatcher(tt.key, tt.prefix, tt.invert) + if got := hpm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestHeaderSuffixMatcherMatch(t *testing.T) { + tests := []struct { + name string + key, suffix string + md metadata.MD + want bool + invert bool + }{ + { + name: "one value one match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv"), + want: true, + }, + { + name: "two value one match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv", "th", "abc"), + want: false, + }, + { + name: "two value match concatenated", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc", "th", "123tv"), + want: true, + }, + { + name: "not match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc"), + want: false, + }, + { + name: "invert header not present", + key: "th", + suffix: "tv", + md: metadata.Pairs(":method", "GET"), + want: false, + invert: true, + }, + { + name: "invert header match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "123tv"), + want: false, + invert: true, + }, + { + name: "invert header not match", + key: "th", + suffix: "tv", + md: metadata.Pairs("th", "abc"), + want: true, + invert: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hsm := NewHeaderSuffixMatcher(tt.key, tt.suffix, tt.invert) + if got := hsm.Match(tt.md); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go new file mode 100644 index 0000000000..ad6d4b283b --- /dev/null +++ b/xds/utils/matcher/regex.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import "regexp" + +// FullMatchWithRegex returns whether the full text matches the regex provided. +func FullMatchWithRegex(re *regexp.Regexp, text string) bool { + if len(text) == 0 { + return re.MatchString(text) + } + re.Longest() + rem := re.FindString(text) + return len(rem) == len(text) +} diff --git a/xds/utils/matcher/regex_test.go b/xds/utils/matcher/regex_test.go new file mode 100644 index 0000000000..b26565336e --- /dev/null +++ b/xds/utils/matcher/regex_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" +) + +func TestFullMatchWithRegex(t *testing.T) { + tests := []struct { + name string + regexStr string + string string + want bool + }{ + { + name: "not match because only partial", + regexStr: "^a+$", + string: "ab", + want: false, + }, + { + name: "match because fully match", + regexStr: "^a+$", + string: "aa", + want: true, + }, + { + name: "longest", + regexStr: "a(|b)", + string: "ab", + want: true, + }, + { + name: "match all", + regexStr: ".*", + string: "", + want: true, + }, + { + name: "matches non-empty strings", + regexStr: ".+", + string: "", + want: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hrm := regexp.MustCompile(tt.regexStr) + if got := FullMatchWithRegex(hrm, tt.string); got != tt.want { + t.Errorf("match() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go new file mode 100644 index 0000000000..9703fd4cac --- /dev/null +++ b/xds/utils/matcher/string_matcher.go @@ -0,0 +1,184 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package matcher contains types that need to be shared between code under +// google.golang.org/grpc/xds/... and the rest of gRPC. +package matcher + +import ( + "errors" + "fmt" + "regexp" + "strings" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcutil" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +// StringMatcher contains match criteria for matching a string, and is an +// internal representation of the `StringMatcher` proto defined at +// https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto. +type StringMatcher struct { + // Since these match fields are part of a `oneof` in the corresponding xDS + // proto, only one of them is expected to be set. + exactMatch *string + prefixMatch *string + suffixMatch *string + regexMatch *regexp.Regexp + containsMatch *string + // If true, indicates the exact/prefix/suffix/contains matching should be + // case insensitive. This has no effect on the regex match. + ignoreCase bool +} + +// Match returns true if input matches the criteria in the given StringMatcher. +func (sm StringMatcher) Match(input string) bool { + if sm.ignoreCase { + input = strings.ToLower(input) + } + switch { + case sm.exactMatch != nil: + return input == *sm.exactMatch + case sm.prefixMatch != nil: + return strings.HasPrefix(input, *sm.prefixMatch) + case sm.suffixMatch != nil: + return strings.HasSuffix(input, *sm.suffixMatch) + case sm.regexMatch != nil: + return grpcutil.FullMatchWithRegex(sm.regexMatch, input) + case sm.containsMatch != nil: + return strings.Contains(input, *sm.containsMatch) + } + return false +} + +// StringMatcherFromProto is a helper function to create a StringMatcher from +// the corresponding StringMatcher proto. +// +// Returns a non-nil error if matcherProto is invalid. +func StringMatcherFromProto(matcherProto *v3matcherpb.StringMatcher) (StringMatcher, error) { + if matcherProto == nil { + return StringMatcher{}, errors.New("input StringMatcher proto is nil") + } + + matcher := StringMatcher{ignoreCase: matcherProto.GetIgnoreCase()} + switch mt := matcherProto.GetMatchPattern().(type) { + case *v3matcherpb.StringMatcher_Exact: + matcher.exactMatch = &mt.Exact + if matcher.ignoreCase { + *matcher.exactMatch = strings.ToLower(*matcher.exactMatch) + } + case *v3matcherpb.StringMatcher_Prefix: + if matcherProto.GetPrefix() == "" { + return StringMatcher{}, errors.New("empty prefix is not allowed in StringMatcher") + } + matcher.prefixMatch = &mt.Prefix + if matcher.ignoreCase { + *matcher.prefixMatch = strings.ToLower(*matcher.prefixMatch) + } + case *v3matcherpb.StringMatcher_Suffix: + if matcherProto.GetSuffix() == "" { + return StringMatcher{}, errors.New("empty suffix is not allowed in StringMatcher") + } + matcher.suffixMatch = &mt.Suffix + if matcher.ignoreCase { + *matcher.suffixMatch = strings.ToLower(*matcher.suffixMatch) + } + case *v3matcherpb.StringMatcher_SafeRegex: + regex := matcherProto.GetSafeRegex().GetRegex() + re, err := regexp.Compile(regex) + if err != nil { + return StringMatcher{}, fmt.Errorf("safe_regex matcher %q is invalid", regex) + } + matcher.regexMatch = re + case *v3matcherpb.StringMatcher_Contains: + if matcherProto.GetContains() == "" { + return StringMatcher{}, errors.New("empty contains is not allowed in StringMatcher") + } + matcher.containsMatch = &mt.Contains + if matcher.ignoreCase { + *matcher.containsMatch = strings.ToLower(*matcher.containsMatch) + } + default: + return StringMatcher{}, fmt.Errorf("unrecognized string matcher: %+v", matcherProto) + } + return matcher, nil +} + +// StringMatcherForTesting is a helper function to create a StringMatcher based +// on the given arguments. Intended only for testing purposes. +func StringMatcherForTesting(exact, prefix, suffix, contains *string, regex *regexp.Regexp, ignoreCase bool) StringMatcher { + sm := StringMatcher{ + exactMatch: exact, + prefixMatch: prefix, + suffixMatch: suffix, + regexMatch: regex, + containsMatch: contains, + ignoreCase: ignoreCase, + } + if ignoreCase { + switch { + case sm.exactMatch != nil: + *sm.exactMatch = strings.ToLower(*exact) + case sm.prefixMatch != nil: + *sm.prefixMatch = strings.ToLower(*prefix) + case sm.suffixMatch != nil: + *sm.suffixMatch = strings.ToLower(*suffix) + case sm.containsMatch != nil: + *sm.containsMatch = strings.ToLower(*contains) + } + } + return sm +} + +// ExactMatch returns the value of the configured exact match or an empty string +// if exact match criteria was not specified. +func (sm StringMatcher) ExactMatch() string { + if sm.exactMatch != nil { + return *sm.exactMatch + } + return "" +} + +// Equal returns true if other and sm are equivalent to each other. +func (sm StringMatcher) Equal(other StringMatcher) bool { + if sm.ignoreCase != other.ignoreCase { + return false + } + + if (sm.exactMatch != nil) != (other.exactMatch != nil) || + (sm.prefixMatch != nil) != (other.prefixMatch != nil) || + (sm.suffixMatch != nil) != (other.suffixMatch != nil) || + (sm.regexMatch != nil) != (other.regexMatch != nil) || + (sm.containsMatch != nil) != (other.containsMatch != nil) { + return false + } + + switch { + case sm.exactMatch != nil: + return *sm.exactMatch == *other.exactMatch + case sm.prefixMatch != nil: + return *sm.prefixMatch == *other.prefixMatch + case sm.suffixMatch != nil: + return *sm.suffixMatch == *other.suffixMatch + case sm.regexMatch != nil: + return sm.regexMatch.String() == other.regexMatch.String() + case sm.containsMatch != nil: + return *sm.containsMatch == *other.containsMatch + } + return true +} diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go new file mode 100644 index 0000000000..9528b57e44 --- /dev/null +++ b/xds/utils/matcher/string_matcher_test.go @@ -0,0 +1,315 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package matcher + +import ( + "regexp" + "testing" + + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/google/go-cmp/cmp" +) + +func TestStringMatcherFromProto(t *testing.T) { + tests := []struct { + desc string + inputProto *v3matcherpb.StringMatcher + wantMatcher StringMatcher + wantErr bool + }{ + { + desc: "nil proto", + wantErr: true, + }, + { + desc: "empty prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: ""}, + }, + wantErr: true, + }, + { + desc: "empty suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: ""}, + }, + wantErr: true, + }, + { + desc: "empty contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: ""}, + }, + wantErr: true, + }, + { + desc: "invalid regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "??"}, + }, + }, + wantErr: true, + }, + { + desc: "happy case exact", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}, + }, + wantMatcher: StringMatcher{exactMatch: newStringP("exact")}, + }, + { + desc: "happy case exact ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "EXACT"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + exactMatch: newStringP("exact"), + ignoreCase: true, + }, + }, + { + desc: "happy case prefix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}, + }, + wantMatcher: StringMatcher{prefixMatch: newStringP("prefix")}, + }, + { + desc: "happy case prefix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "PREFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + prefixMatch: newStringP("prefix"), + ignoreCase: true, + }, + }, + { + desc: "happy case suffix", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}, + }, + wantMatcher: StringMatcher{suffixMatch: newStringP("suffix")}, + }, + { + desc: "happy case suffix ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "SUFFIX"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + suffixMatch: newStringP("suffix"), + ignoreCase: true, + }, + }, + { + desc: "happy case regex", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{ + SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}, + }, + }, + wantMatcher: StringMatcher{regexMatch: regexp.MustCompile("good?regex?")}, + }, + { + desc: "happy case contains", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}, + }, + wantMatcher: StringMatcher{containsMatch: newStringP("contains")}, + }, + { + desc: "happy case contains ignore case", + inputProto: &v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "CONTAINS"}, + IgnoreCase: true, + }, + wantMatcher: StringMatcher{ + containsMatch: newStringP("contains"), + ignoreCase: true, + }, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + gotMatcher, err := StringMatcherFromProto(test.inputProto) + if (err != nil) != test.wantErr { + t.Fatalf("StringMatcherFromProto(%+v) returned err: %v, wantErr: %v", test.inputProto, err, test.wantErr) + } + if diff := cmp.Diff(gotMatcher, test.wantMatcher, cmp.AllowUnexported(regexp.Regexp{})); diff != "" { + t.Fatalf("StringMatcherFromProto(%+v) returned unexpected diff (-got, +want):\n%s", test.inputProto, diff) + } + }) + } +} + +func TestMatch(t *testing.T) { + var ( + exactMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}}) + prefixMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}}) + suffixMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}}) + regexMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_SafeRegex{SafeRegex: &v3matcherpb.RegexMatcher{Regex: "good?regex?"}}}) + containsMatcher, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}}) + exactMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Exact{Exact: "exact"}, + IgnoreCase: true, + }) + prefixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Prefix{Prefix: "prefix"}, + IgnoreCase: true, + }) + suffixMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Suffix{Suffix: "suffix"}, + IgnoreCase: true, + }) + containsMatcherIgnoreCase, _ = StringMatcherFromProto(&v3matcherpb.StringMatcher{ + MatchPattern: &v3matcherpb.StringMatcher_Contains{Contains: "contains"}, + IgnoreCase: true, + }) + ) + + tests := []struct { + desc string + matcher StringMatcher + input string + wantMatch bool + }{ + { + desc: "exact match success", + matcher: exactMatcher, + input: "exact", + wantMatch: true, + }, + { + desc: "exact match failure", + matcher: exactMatcher, + input: "not-exact", + }, + { + desc: "exact match success with ignore case", + matcher: exactMatcherIgnoreCase, + input: "EXACT", + wantMatch: true, + }, + { + desc: "exact match failure with ignore case", + matcher: exactMatcherIgnoreCase, + input: "not-exact", + }, + { + desc: "prefix match success", + matcher: prefixMatcher, + input: "prefixIsHere", + wantMatch: true, + }, + { + desc: "prefix match failure", + matcher: prefixMatcher, + input: "not-prefix", + }, + { + desc: "prefix match success with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "PREFIXisHere", + wantMatch: true, + }, + { + desc: "prefix match failure with ignore case", + matcher: prefixMatcherIgnoreCase, + input: "not-PREFIX", + }, + { + desc: "suffix match success", + matcher: suffixMatcher, + input: "hereIsThesuffix", + wantMatch: true, + }, + { + desc: "suffix match failure", + matcher: suffixMatcher, + input: "suffix-is-not-here", + }, + { + desc: "suffix match success with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "hereIsTheSuFFix", + wantMatch: true, + }, + { + desc: "suffix match failure with ignore case", + matcher: suffixMatcherIgnoreCase, + input: "SUFFIX-is-not-here", + }, + { + desc: "regex match success", + matcher: regexMatcher, + input: "goodregex", + wantMatch: true, + }, + { + desc: "regex match failure because only part match", + matcher: regexMatcher, + input: "goodregexa", + wantMatch: false, + }, + { + desc: "regex match failure", + matcher: regexMatcher, + input: "regex-is-not-here", + }, + { + desc: "contains match success", + matcher: containsMatcher, + input: "IScontainsHERE", + wantMatch: true, + }, + { + desc: "contains match failure", + matcher: containsMatcher, + input: "con-tains-is-not-here", + }, + { + desc: "contains match success with ignore case", + matcher: containsMatcherIgnoreCase, + input: "isCONTAINShere", + wantMatch: true, + }, + { + desc: "contains match failure with ignore case", + matcher: containsMatcherIgnoreCase, + input: "CON-TAINS-is-not-here", + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + if gotMatch := test.matcher.Match(test.input); gotMatch != test.wantMatch { + t.Errorf("StringMatcher.Match(%s) returned %v, want %v", test.input, gotMatch, test.wantMatch) + } + }) + } +} + +func newStringP(s string) *string { + return &s +} diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go new file mode 100644 index 0000000000..9ebb1b22d5 --- /dev/null +++ b/xds/utils/metadata/metadata.go @@ -0,0 +1,40 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package metadata + +import ( + "context" + + "google.golang.org/grpc/metadata" +) + +type mdExtraKey struct{} + +// WithExtraMetadata creates a new context with incoming md attached. +func WithExtraMetadata(ctx context.Context, md metadata.MD) context.Context { + return context.WithValue(ctx, mdExtraKey{}, md) +} + +// ExtraMetadata returns the incoming metadata in ctx if it exists. The +// returned MD should not be modified. Writing to it may cause races. +// Modification should be made to copies of the returned MD. +func ExtraMetadata(ctx context.Context) (md metadata.MD, ok bool) { + md, ok = ctx.Value(mdExtraKey{}).(metadata.MD) + return +} diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go new file mode 100644 index 0000000000..0177af4b51 --- /dev/null +++ b/xds/utils/pretty/pretty.go @@ -0,0 +1,82 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package pretty defines helper functions to pretty-print structs for logging. +package pretty + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/golang/protobuf/jsonpb" + protov1 "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/encoding/protojson" + protov2 "google.golang.org/protobuf/proto" +) + +const jsonIndent = " " + +// ToJSON marshals the input into a json string. +// +// If marshal fails, it falls back to fmt.Sprintf("%+v"). +func ToJSON(e interface{}) string { + switch ee := e.(type) { + case protov1.Message: + mm := jsonpb.Marshaler{Indent: jsonIndent} + ret, err := mm.MarshalToString(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return ret + case protov2.Message: + mm := protojson.MarshalOptions{ + Multiline: true, + Indent: jsonIndent, + } + ret, err := mm.Marshal(ee) + if err != nil { + // This may fail for proto.Anys, e.g. for xDS v2, LDS, the v2 + // messages are not imported, and this will fail because the message + // is not found. + return fmt.Sprintf("%+v", ee) + } + return string(ret) + default: + ret, err := json.MarshalIndent(ee, "", jsonIndent) + if err != nil { + return fmt.Sprintf("%+v", ee) + } + return string(ret) + } +} + +// FormatJSON formats the input json bytes with indentation. +// +// If Indent fails, it returns the unchanged input as string. +func FormatJSON(b []byte) string { + var out bytes.Buffer + err := json.Indent(&out, b, "", jsonIndent) + if err != nil { + return string(b) + } + return out.String() +} diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go new file mode 100644 index 0000000000..edb39ea92b --- /dev/null +++ b/xds/utils/rbac/matchers.go @@ -0,0 +1,423 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package rbac + +import ( + "errors" + "fmt" + "net" + "regexp" + + internalmatcher "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + v3route_componentspb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" +) + +// matcher is an interface that takes data about incoming RPC's and returns +// whether it matches with whatever matcher implements this interface. +type matcher interface { + match(data *rpcData) bool +} + +// policyMatcher helps determine whether an incoming RPC call matches a policy. +// A policy is a logical role (e.g. Service Admin), which is comprised of +// permissions and principals. A principal is an identity (or identities) for a +// downstream subject which are assigned the policy (role), and a permission is +// an action(s) that a principal(s) can take. A policy matches if both a +// permission and a principal match, which will be determined by the child or +// permissions and principal matchers. policyMatcher implements the matcher +// interface. +type policyMatcher struct { + permissions *orMatcher + principals *orMatcher +} + +func newPolicyMatcher(policy *v3rbacpb.Policy) (*policyMatcher, error) { + permissions, err := matchersFromPermissions(policy.Permissions) + if err != nil { + return nil, err + } + principals, err := matchersFromPrincipals(policy.Principals) + if err != nil { + return nil, err + } + return &policyMatcher{ + permissions: &orMatcher{matchers: permissions}, + principals: &orMatcher{matchers: principals}, + }, nil +} + +func (pm *policyMatcher) match(data *rpcData) bool { + // A policy matches if and only if at least one of its permissions match the + // action taking place AND at least one if its principals match the + // downstream peer. + return pm.permissions.match(data) && pm.principals.match(data) +} + +// matchersFromPermissions takes a list of permissions (can also be +// a single permission, e.g. from a not matcher which is logically !permission) +// and returns a list of matchers which correspond to that permission. This will +// be called in many instances throughout the initial construction of the RBAC +// engine from the AND and OR matchers and also from the NOT matcher. +func matchersFromPermissions(permissions []*v3rbacpb.Permission) ([]matcher, error) { + var matchers []matcher + for _, permission := range permissions { + switch permission.GetRule().(type) { + case *v3rbacpb.Permission_AndRules: + mList, err := matchersFromPermissions(permission.GetAndRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Permission_OrRules: + mList, err := matchersFromPermissions(permission.GetOrRules().Rules) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Permission_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Permission_Header: + m, err := newHeaderMatcher(permission.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_UrlPath: + m, err := newURLPathMatcher(permission.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationIp: + // Due to this being on server side, the destination IP is the local + // IP. + m, err := newLocalIPMatcher(permission.GetDestinationIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Permission_DestinationPort: + matchers = append(matchers, newPortMatcher(permission.GetDestinationPort())) + case *v3rbacpb.Permission_NotRule: + mList, err := matchersFromPermissions([]*v3rbacpb.Permission{{Rule: permission.GetNotRule().Rule}}) + if err != nil { + return nil, err + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Permission_Metadata: + // Not supported in gRPC RBAC currently - a permission typed as + // Metadata in the initial config will be a no-op. + case *v3rbacpb.Permission_RequestedServerName: + // Not supported in gRPC RBAC currently - a permission typed as + // requested server name in the initial config will be a no-op. + } + } + return matchers, nil +} + +func matchersFromPrincipals(principals []*v3rbacpb.Principal) ([]matcher, error) { + var matchers []matcher + for _, principal := range principals { + switch principal.GetIdentifier().(type) { + case *v3rbacpb.Principal_AndIds: + mList, err := matchersFromPrincipals(principal.GetAndIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &andMatcher{matchers: mList}) + case *v3rbacpb.Principal_OrIds: + mList, err := matchersFromPrincipals(principal.GetOrIds().Ids) + if err != nil { + return nil, err + } + matchers = append(matchers, &orMatcher{matchers: mList}) + case *v3rbacpb.Principal_Any: + matchers = append(matchers, &alwaysMatcher{}) + case *v3rbacpb.Principal_Authenticated_: + authenticatedMatcher, err := newAuthenticatedMatcher(principal.GetAuthenticated()) + if err != nil { + return nil, err + } + matchers = append(matchers, authenticatedMatcher) + case *v3rbacpb.Principal_DirectRemoteIp: + m, err := newRemoteIPMatcher(principal.GetDirectRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Header: + // Do we need an error here? + m, err := newHeaderMatcher(principal.GetHeader()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_UrlPath: + m, err := newURLPathMatcher(principal.GetUrlPath()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_NotId: + mList, err := matchersFromPrincipals([]*v3rbacpb.Principal{{Identifier: principal.GetNotId().Identifier}}) + if err != nil { + return nil, err + } + matchers = append(matchers, ¬Matcher{matcherToNot: mList[0]}) + case *v3rbacpb.Principal_SourceIp: + // The source ip principal identifier is deprecated. Thus, a + // principal typed as a source ip in the identifier will be a no-op. + // The config should use DirectRemoteIp instead. + case *v3rbacpb.Principal_RemoteIp: + // RBAC in gRPC treats direct_remote_ip and remote_ip as logically + // equivalent, as per A41. + m, err := newRemoteIPMatcher(principal.GetRemoteIp()) + if err != nil { + return nil, err + } + matchers = append(matchers, m) + case *v3rbacpb.Principal_Metadata: + // Not supported in gRPC RBAC currently - a principal typed as + // Metadata in the initial config will be a no-op. + } + } + return matchers, nil +} + +// orMatcher is a matcher where it successfully matches if one of it's +// children successfully match. It also logically represents a principal or +// permission, but can also be it's own entity further down the tree of +// matchers. orMatcher implements the matcher interface. +type orMatcher struct { + matchers []matcher +} + +func (om *orMatcher) match(data *rpcData) bool { + // Range through child matchers and pass in data about incoming RPC, and + // only one child matcher has to match to be logically successful. + for _, m := range om.matchers { + if m.match(data) { + return true + } + } + return false +} + +// andMatcher is a matcher that is successful if every child matcher +// matches. andMatcher implements the matcher interface. +type andMatcher struct { + matchers []matcher +} + +func (am *andMatcher) match(data *rpcData) bool { + for _, m := range am.matchers { + if !m.match(data) { + return false + } + } + return true +} + +// alwaysMatcher is a matcher that will always match. This logically +// represents an any rule for a permission or a principal. alwaysMatcher +// implements the matcher interface. +type alwaysMatcher struct { +} + +func (am *alwaysMatcher) match(data *rpcData) bool { + return true +} + +// notMatcher is a matcher that nots an underlying matcher. notMatcher +// implements the matcher interface. +type notMatcher struct { + matcherToNot matcher +} + +func (nm *notMatcher) match(data *rpcData) bool { + return !nm.matcherToNot.match(data) +} + +// headerMatcher is a matcher that matches on incoming HTTP Headers present +// in the incoming RPC. headerMatcher implements the matcher interface. +type headerMatcher struct { + matcher internalmatcher.HeaderMatcher +} + +func newHeaderMatcher(headerMatcherConfig *v3route_componentspb.HeaderMatcher) (*headerMatcher, error) { + var m internalmatcher.HeaderMatcher + switch headerMatcherConfig.HeaderMatchSpecifier.(type) { + case *v3route_componentspb.HeaderMatcher_ExactMatch: + m = internalmatcher.NewHeaderExactMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetExactMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SafeRegexMatch: + regex, err := regexp.Compile(headerMatcherConfig.GetSafeRegexMatch().Regex) + if err != nil { + return nil, err + } + m = internalmatcher.NewHeaderRegexMatcher(headerMatcherConfig.Name, regex, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_RangeMatch: + m = internalmatcher.NewHeaderRangeMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetRangeMatch().Start, headerMatcherConfig.GetRangeMatch().End, headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PresentMatch: + m = internalmatcher.NewHeaderPresentMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPresentMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_PrefixMatch: + m = internalmatcher.NewHeaderPrefixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetPrefixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_SuffixMatch: + m = internalmatcher.NewHeaderSuffixMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetSuffixMatch(), headerMatcherConfig.InvertMatch) + case *v3route_componentspb.HeaderMatcher_ContainsMatch: + m = internalmatcher.NewHeaderContainsMatcher(headerMatcherConfig.Name, headerMatcherConfig.GetContainsMatch(), headerMatcherConfig.InvertMatch) + default: + return nil, errors.New("unknown header matcher type") + } + return &headerMatcher{matcher: m}, nil +} + +func (hm *headerMatcher) match(data *rpcData) bool { + return hm.matcher.Match(data.md) +} + +// urlPathMatcher matches on the URL Path of the incoming RPC. In gRPC, this +// logically maps to the full method name the RPC is calling on the server side. +// urlPathMatcher implements the matcher interface. +type urlPathMatcher struct { + stringMatcher internalmatcher.StringMatcher +} + +func newURLPathMatcher(pathMatcher *v3matcherpb.PathMatcher) (*urlPathMatcher, error) { + stringMatcher, err := internalmatcher.StringMatcherFromProto(pathMatcher.GetPath()) + if err != nil { + return nil, err + } + return &urlPathMatcher{stringMatcher: stringMatcher}, nil +} + +func (upm *urlPathMatcher) match(data *rpcData) bool { + return upm.stringMatcher.Match(data.fullMethod) +} + +// remoteIPMatcher and localIPMatcher both are matchers that match against +// a CIDR Range. Two different matchers are needed as the remote and destination +// ip addresses come from different parts of the data about incoming RPC's +// passed in. Matching a CIDR Range means to determine whether the IP Address +// falls within the CIDR Range or not. They both implement the matcher +// interface. +type remoteIPMatcher struct { + // ipNet represents the CidrRange that this matcher was configured with. + // This is what will remote and destination IP's will be matched against. + ipNet *net.IPNet +} + +func newRemoteIPMatcher(cidrRange *v3corepb.CidrRange) (*remoteIPMatcher, error) { + // Convert configuration to a cidrRangeString, as Go standard library has + // methods that parse cidr string. + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.AddressPrefix, cidrRange.PrefixLen.Value) + _, ipNet, err := net.ParseCIDR(cidrRangeString) + if err != nil { + return nil, err + } + return &remoteIPMatcher{ipNet: ipNet}, nil +} + +func (sim *remoteIPMatcher) match(data *rpcData) bool { + return sim.ipNet.Contains(net.IP(net.ParseIP(data.peerInfo.Addr.String()))) +} + +type localIPMatcher struct { + ipNet *net.IPNet +} + +func newLocalIPMatcher(cidrRange *v3corepb.CidrRange) (*localIPMatcher, error) { + cidrRangeString := fmt.Sprintf("%s/%d", cidrRange.AddressPrefix, cidrRange.PrefixLen.Value) + _, ipNet, err := net.ParseCIDR(cidrRangeString) + if err != nil { + return nil, err + } + return &localIPMatcher{ipNet: ipNet}, nil +} + +func (dim *localIPMatcher) match(data *rpcData) bool { + return dim.ipNet.Contains(net.IP(net.ParseIP(data.localAddr.String()))) +} + +// portMatcher matches on whether the destination port of the RPC matches the +// destination port this matcher was instantiated with. portMatcher +// implements the matcher interface. +type portMatcher struct { + destinationPort uint32 +} + +func newPortMatcher(destinationPort uint32) *portMatcher { + return &portMatcher{destinationPort: destinationPort} +} + +func (pm *portMatcher) match(data *rpcData) bool { + return data.destinationPort == pm.destinationPort +} + +// authenticatedMatcher matches on the name of the Principal. If set, the URI +// SAN or DNS SAN in that order is used from the certificate, otherwise the +// subject field is used. If unset, it applies to any user that is +// authenticated. authenticatedMatcher implements the matcher interface. +type authenticatedMatcher struct { + stringMatcher *internalmatcher.StringMatcher +} + +func newAuthenticatedMatcher(authenticatedMatcherConfig *v3rbacpb.Principal_Authenticated) (*authenticatedMatcher, error) { + // Represents this line in the RBAC documentation = "If unset, it applies to + // any user that is authenticated" (see package-level comments). + if authenticatedMatcherConfig.PrincipalName == nil { + return &authenticatedMatcher{}, nil + } + stringMatcher, err := internalmatcher.StringMatcherFromProto(authenticatedMatcherConfig.PrincipalName) + if err != nil { + return nil, err + } + return &authenticatedMatcher{stringMatcher: &stringMatcher}, nil +} + +func (am *authenticatedMatcher) match(data *rpcData) bool { + if data.authType != "tls" { + // Connection is not authenticated. + return false + } + if am.stringMatcher == nil { + // Allows any authenticated user. + return true + } + // "If there is no client certificate (thus no SAN nor Subject), check if "" + // (empty string) matches. If it matches, the principal_name is said to + // match" - A41 + if len(data.certs) == 0 { + return am.stringMatcher.Match("") + } + cert := data.certs[0] + // The order of matching as per the RBAC documentation (see package-level comments) + // is as follows: URI SANs, DNS SANs, and then subject name. + for _, uriSAN := range cert.URIs { + if am.stringMatcher.Match(uriSAN.String()) { + return true + } + } + for _, dnsSAN := range cert.DNSNames { + if am.stringMatcher.Match(dnsSAN) { + return true + } + } + return am.stringMatcher.Match(cert.Subject.String()) +} diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go new file mode 100644 index 0000000000..3ed7895c03 --- /dev/null +++ b/xds/utils/rbac/rbac_engine.go @@ -0,0 +1,230 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package rbac provides service-level and method-level access control for a +// service. See +// https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/rbac/v3/rbac.proto#role-based-access-control-rbac +// for documentation. +package rbac + +import ( + "context" + "crypto/x509" + "errors" + "fmt" + "net" + "strconv" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/peer" + "google.golang.org/grpc/status" +) + +const logLevel = 2 + +var logger = grpclog.Component("rbac") + +var getConnection = transport.GetConnection + +// ChainEngine represents a chain of RBAC Engines, used to make authorization +// decisions on incoming RPCs. +type ChainEngine struct { + chainedEngines []*engine +} + +// NewChainEngine returns a chain of RBAC engines, used to make authorization +// decisions on incoming RPCs. Returns a non-nil error for invalid policies. +func NewChainEngine(policies []*v3rbacpb.RBAC) (*ChainEngine, error) { + engines := make([]*engine, 0, len(policies)) + for _, policy := range policies { + engine, err := newEngine(policy) + if err != nil { + return nil, err + } + engines = append(engines, engine) + } + return &ChainEngine{chainedEngines: engines}, nil +} + +// IsAuthorized determines if an incoming RPC is authorized based on the chain of RBAC +// engines and their associated actions. +// +// Errors returned by this function are compatible with the status package. +func (cre *ChainEngine) IsAuthorized(ctx context.Context) error { + // This conversion step (i.e. pulling things out of ctx) can be done once, + // and then be used for the whole chain of RBAC Engines. + rpcData, err := newRPCData(ctx) + if err != nil { + logger.Errorf("newRPCData: %v", err) + return status.Errorf(codes.Internal, "gRPC RBAC: %v", err) + } + for _, engine := range cre.chainedEngines { + matchingPolicyName, ok := engine.findMatchingPolicy(rpcData) + if logger.V(logLevel) && ok { + logger.Infof("incoming RPC matched to policy %v in engine with action %v", matchingPolicyName, engine.action) + } + + switch { + case engine.action == v3rbacpb.RBAC_ALLOW && !ok: + return status.Errorf(codes.PermissionDenied, "incoming RPC did not match an allow policy") + case engine.action == v3rbacpb.RBAC_DENY && ok: + return status.Errorf(codes.PermissionDenied, "incoming RPC matched a deny policy %q", matchingPolicyName) + } + // Every policy in the engine list must be queried. Thus, iterate to the + // next policy. + } + // If the incoming RPC gets through all of the engines successfully (i.e. + // doesn't not match an allow or match a deny engine), the RPC is authorized + // to proceed. + return nil +} + +// engine is used for matching incoming RPCs to policies. +type engine struct { + policies map[string]*policyMatcher + // action must be ALLOW or DENY. + action v3rbacpb.RBAC_Action +} + +// newEngine creates an RBAC Engine based on the contents of policy. Returns a +// non-nil error if the policy is invalid. +func newEngine(config *v3rbacpb.RBAC) (*engine, error) { + a := config.GetAction() + if a != v3rbacpb.RBAC_ALLOW && a != v3rbacpb.RBAC_DENY { + return nil, fmt.Errorf("unsupported action %s", config.Action) + } + + policies := make(map[string]*policyMatcher, len(config.GetPolicies())) + for name, policy := range config.GetPolicies() { + matcher, err := newPolicyMatcher(policy) + if err != nil { + return nil, err + } + policies[name] = matcher + } + return &engine{ + policies: policies, + action: a, + }, nil +} + +// findMatchingPolicy determines if an incoming RPC matches a policy. On a +// successful match, it returns the name of the matching policy and a true bool +// to specify that there was a matching policy found. It returns false in +// the case of not finding a matching policy. +func (r *engine) findMatchingPolicy(rpcData *rpcData) (string, bool) { + for policy, matcher := range r.policies { + if matcher.match(rpcData) { + return policy, true + } + } + return "", false +} + +// newRPCData takes an incoming context (should be a context representing state +// needed for server RPC Call with metadata, peer info (used for source ip/port +// and TLS information) and connection (used for destination ip/port) piped into +// it) and the method name of the Service being called server side and populates +// an rpcData struct ready to be passed to the RBAC Engine to find a matching +// policy. +func newRPCData(ctx context.Context) (*rpcData, error) { + // The caller should populate all of these fields (i.e. for empty headers, + // pipe an empty md into context). + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil, errors.New("missing metadata in incoming context") + } + // ":method can be hard-coded to POST if unavailable" - A41 + md[":method"] = []string{"POST"} + // "If the transport exposes TE in Metadata, then RBAC must special-case the + // header to treat it as not present." - A41 + delete(md, "TE") + + pi, ok := peer.FromContext(ctx) + if !ok { + return nil, errors.New("missing peer info in incoming context") + } + + // The methodName will be available in the passed in ctx from a unary or streaming + // interceptor, as grpc.Server pipes in a transport stream which contains the methodName + // into contexts available in both unary or streaming interceptors. + mn, ok := grpc.Method(ctx) + if !ok { + return nil, errors.New("missing method in incoming context") + } + + // The connection is needed in order to find the destination address and + // port of the incoming RPC Call. + conn := getConnection(ctx) + if conn == nil { + return nil, errors.New("missing connection in incoming context") + } + _, dPort, err := net.SplitHostPort(conn.LocalAddr().String()) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + dp, err := strconv.ParseUint(dPort, 10, 32) + if err != nil { + return nil, fmt.Errorf("error parsing local address: %v", err) + } + + var authType string + var peerCertificates []*x509.Certificate + if pi.AuthInfo != nil { + tlsInfo, ok := pi.AuthInfo.(credentials.TLSInfo) + if ok { + authType = pi.AuthInfo.AuthType() + peerCertificates = tlsInfo.State.PeerCertificates + } + } + + return &rpcData{ + md: md, + peerInfo: pi, + fullMethod: mn, + destinationPort: uint32(dp), + localAddr: conn.LocalAddr(), + authType: authType, + certs: peerCertificates, + }, nil +} + +// rpcData wraps data pulled from an incoming RPC that the RBAC engine needs to +// find a matching policy. +type rpcData struct { + // md is the HTTP Headers that are present in the incoming RPC. + md metadata.MD + // peerInfo is information about the downstream peer. + peerInfo *peer.Peer + // fullMethod is the method name being called on the upstream service. + fullMethod string + // destinationPort is the port that the RPC is being sent to on the + // server. + destinationPort uint32 + // localAddr is the address that the RPC is being sent to. + localAddr net.Addr + // authType is the type of authentication e.g. "tls". + authType string + // certs are the certificates presented by the peer during a TLS + // handshake. + certs []*x509.Certificate +} diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go new file mode 100644 index 0000000000..26417daad6 --- /dev/null +++ b/xds/utils/resolver/config_selector.go @@ -0,0 +1,167 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package resolver provides internal resolver-related functionality. +package resolver + +import ( + "context" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" +) + +// ConfigSelector controls what configuration to use for every RPC. +type ConfigSelector interface { + // Selects the configuration for the RPC, or terminates it using the error. + // This error will be converted by the gRPC library to a status error with + // code UNKNOWN if it is not returned as a status error. + SelectConfig(RPCInfo) (*RPCConfig, error) +} + +// RPCInfo contains RPC information needed by a ConfigSelector. +type RPCInfo struct { + // Context is the user's context for the RPC and contains headers and + // application timeout. It is passed for interception purposes and for + // efficiency reasons. SelectConfig should not be blocking. + Context context.Context + Method string // i.e. "/Service/Method" +} + +// RPCConfig describes the configuration to use for each RPC. +type RPCConfig struct { + // The context to use for the remainder of the RPC; can pass info to LB + // policy or affect timeout or metadata. + Context context.Context + MethodConfig serviceconfig.MethodConfig // configuration to use for this RPC + OnCommitted func() // Called when the RPC has been committed (retries no longer possible) + Interceptor ClientInterceptor +} + +// ClientStream is the same as grpc.ClientStream, but defined here for circular +// dependency reasons. +type ClientStream interface { + // Header returns the header metadata received from the server if there + // is any. It blocks if the metadata is not ready to read. + Header() (metadata.MD, error) + // Trailer returns the trailer metadata from the server, if there is any. + // It must only be called after stream.CloseAndRecv has returned, or + // stream.Recv has returned a non-nil error (including io.EOF). + Trailer() metadata.MD + // CloseSend closes the send direction of the stream. It closes the stream + // when non-nil error is met. It is also not safe to call CloseSend + // concurrently with SendMsg. + CloseSend() error + // Context returns the context for this stream. + // + // It should not be called until after Header or RecvMsg has returned. Once + // called, subsequent client-side retries are disabled. + Context() context.Context + // SendMsg is generally called by generated code. On error, SendMsg aborts + // the stream. If the error was generated by the client, the status is + // returned directly; otherwise, io.EOF is returned and the status of + // the stream may be discovered using RecvMsg. + // + // SendMsg blocks until: + // - There is sufficient flow control to schedule m with the transport, or + // - The stream is done, or + // - The stream breaks. + // + // SendMsg does not wait until the message is received by the server. An + // untimely stream closure may result in lost messages. To ensure delivery, + // users should ensure the RPC completed successfully using RecvMsg. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not safe + // to call SendMsg on the same stream in different goroutines. It is also + // not safe to call CloseSend concurrently with SendMsg. + SendMsg(m interface{}) error + // RecvMsg blocks until it receives a message into m or the stream is + // done. It returns io.EOF when the stream completes successfully. On + // any other error, the stream is aborted and the error contains the RPC + // status. + // + // It is safe to have a goroutine calling SendMsg and another goroutine + // calling RecvMsg on the same stream at the same time, but it is not + // safe to call RecvMsg on the same stream in different goroutines. + RecvMsg(m interface{}) error +} + +// ClientInterceptor is an interceptor for gRPC client streams. +type ClientInterceptor interface { + // NewStream produces a ClientStream for an RPC which may optionally use + // the provided function to produce a stream for delegation. Note: + // RPCInfo.Context should not be used (will be nil). + // + // done is invoked when the RPC is finished using its connection, or could + // not be assigned a connection. RPC operations may still occur on + // ClientStream after done is called, since the interceptor is invoked by + // application-layer operations. done must never be nil when called. + NewStream(ctx context.Context, ri RPCInfo, done func(), newStream func(ctx context.Context, done func()) (ClientStream, error)) (ClientStream, error) +} + +// ServerInterceptor is an interceptor for incoming RPC's on gRPC server side. +type ServerInterceptor interface { + // AllowRPC checks if an incoming RPC is allowed to proceed based on + // information about connection RPC was received on, and HTTP Headers. This + // information will be piped into context. + AllowRPC(ctx context.Context) error // TODO: Make this a real interceptor for filters such as rate limiting. +} + +type csKeyType string + +const csKey = csKeyType("grpc.internal.resolver.configSelector") + +// SetConfigSelector sets the config selector in state and returns the new +// state. +func SetConfigSelector(state resolver.State, cs ConfigSelector) resolver.State { + state.Attributes = state.Attributes.WithValue(csKey, cs) + return state +} + +// GetConfigSelector retrieves the config selector from state, if present, and +// returns it or nil if absent. +func GetConfigSelector(state resolver.State) ConfigSelector { + cs, _ := state.Attributes.Value(csKey).(ConfigSelector) + return cs +} + +// SafeConfigSelector allows for safe switching of ConfigSelector +// implementations such that previous values are guaranteed to not be in use +// when UpdateConfigSelector returns. +type SafeConfigSelector struct { + mu sync.RWMutex + cs ConfigSelector +} + +// UpdateConfigSelector swaps to the provided ConfigSelector and blocks until +// all uses of the previous ConfigSelector have completed. +func (scs *SafeConfigSelector) UpdateConfigSelector(cs ConfigSelector) { + scs.mu.Lock() + defer scs.mu.Unlock() + scs.cs = cs +} + +// SelectConfig defers to the current ConfigSelector in scs. +func (scs *SafeConfigSelector) SelectConfig(r RPCInfo) (*RPCConfig, error) { + scs.mu.RLock() + defer scs.mu.RUnlock() + return scs.cs.SelectConfig(r) +} diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go new file mode 100644 index 0000000000..520d9229e1 --- /dev/null +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -0,0 +1,57 @@ +/* + * + * Copyright 2017 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package passthrough implements a pass-through resolver. It sends the target +// name without scheme back to gRPC as resolved address. +package passthrough + +import "google.golang.org/grpc/resolver" + +const scheme = "passthrough" + +type passthroughBuilder struct{} + +func (*passthroughBuilder) Build(target resolver.Target, cc resolver.ClientConn, opts resolver.BuildOptions) (resolver.Resolver, error) { + r := &passthroughResolver{ + target: target, + cc: cc, + } + r.start() + return r, nil +} + +func (*passthroughBuilder) Scheme() string { + return scheme +} + +type passthroughResolver struct { + target resolver.Target + cc resolver.ClientConn +} + +func (r *passthroughResolver) start() { + r.cc.UpdateState(resolver.State{Addresses: []resolver.Address{{Addr: r.target.Endpoint}}}) +} + +func (*passthroughResolver) ResolveNow(o resolver.ResolveNowOptions) {} + +func (*passthroughResolver) Close() {} + +func init() { + resolver.Register(&passthroughBuilder{}) +} diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go new file mode 100644 index 0000000000..10e2ee7222 --- /dev/null +++ b/xds/utils/resolver/unix/unix.go @@ -0,0 +1,73 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package unix implements a resolver for unix targets. +package unix + +import ( + "fmt" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport/networktype" + "google.golang.org/grpc/resolver" +) + +const unixScheme = "unix" +const unixAbstractScheme = "unix-abstract" + +type builder struct { + scheme string +} + +func (b *builder) Build(target resolver.Target, cc resolver.ClientConn, _ resolver.BuildOptions) (resolver.Resolver, error) { + if target.Authority != "" { + return nil, fmt.Errorf("invalid (non-empty) authority: %v", target.Authority) + } + + // gRPC was parsing the dial target manually before PR #4817, and we + // switched to using url.Parse() in that PR. To avoid breaking existing + // resolver implementations we ended up stripping the leading "/" from the + // endpoint. This obviously does not work for the "unix" scheme. Hence we + // end up using the parsed URL instead. + endpoint := target.URL.Path + if endpoint == "" { + endpoint = target.URL.Opaque + } + addr := resolver.Address{Addr: endpoint} + if b.scheme == unixAbstractScheme { + // prepend "\x00" to address for unix-abstract + addr.Addr = "\x00" + addr.Addr + } + cc.UpdateState(resolver.State{Addresses: []resolver.Address{networktype.Set(addr, "unix")}}) + return &nopResolver{}, nil +} + +func (b *builder) Scheme() string { + return b.scheme +} + +type nopResolver struct { +} + +func (*nopResolver) ResolveNow(resolver.ResolveNowOptions) {} + +func (*nopResolver) Close() {} + +func init() { + resolver.Register(&builder{scheme: unixScheme}) + resolver.Register(&builder{scheme: unixAbstractScheme}) +} diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go new file mode 100644 index 0000000000..badbdbf597 --- /dev/null +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -0,0 +1,180 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package serviceconfig contains utility functions to parse service config. +package serviceconfig + +import ( + "encoding/json" + "fmt" + "time" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + externalserviceconfig "google.golang.org/grpc/serviceconfig" +) + +var logger = grpclog.Component("core") + +// BalancerConfig wraps the name and config associated with one load balancing +// policy. It corresponds to a single entry of the loadBalancingConfig field +// from ServiceConfig. +// +// It implements the json.Unmarshaler interface. +// +// https://github.com/grpc/grpc-proto/blob/54713b1e8bc6ed2d4f25fb4dff527842150b91b2/grpc/service_config/service_config.proto#L247 +type BalancerConfig struct { + Name string + Config externalserviceconfig.LoadBalancingConfig +} + +type intermediateBalancerConfig []map[string]json.RawMessage + +// MarshalJSON implements the json.Marshaler interface. +// +// It marshals the balancer and config into a length-1 slice +// ([]map[string]config). +func (bc *BalancerConfig) MarshalJSON() ([]byte, error) { + if bc.Config == nil { + // If config is nil, return empty config `{}`. + return []byte(fmt.Sprintf(`[{%q: %v}]`, bc.Name, "{}")), nil + } + c, err := json.Marshal(bc.Config) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf(`[{%q: %s}]`, bc.Name, c)), nil +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// +// ServiceConfig contains a list of loadBalancingConfigs, each with a name and +// config. This method iterates through that list in order, and stops at the +// first policy that is supported. +// - If the config for the first supported policy is invalid, the whole service +// config is invalid. +// - If the list doesn't contain any supported policy, the whole service config +// is invalid. +func (bc *BalancerConfig) UnmarshalJSON(b []byte) error { + var ir intermediateBalancerConfig + err := json.Unmarshal(b, &ir) + if err != nil { + return err + } + + var names []string + for i, lbcfg := range ir { + if len(lbcfg) != 1 { + return fmt.Errorf("invalid loadBalancingConfig: entry %v does not contain exactly 1 policy/config pair: %q", i, lbcfg) + } + + var ( + name string + jsonCfg json.RawMessage + ) + // Get the key:value pair from the map. We have already made sure that + // the map contains a single entry. + for name, jsonCfg = range lbcfg { + } + + names = append(names, name) + builder := balancer.Get(name) + if builder == nil { + // If the balancer is not registered, move on to the next config. + // This is not an error. + continue + } + bc.Name = name + + parser, ok := builder.(balancer.ConfigParser) + if !ok { + if string(jsonCfg) != "{}" { + logger.Warningf("non-empty balancer configuration %q, but balancer does not implement ParseConfig", string(jsonCfg)) + } + // Stop at this, though the builder doesn't support parsing config. + return nil + } + + cfg, err := parser.ParseConfig(jsonCfg) + if err != nil { + return fmt.Errorf("error parsing loadBalancingConfig for policy %q: %v", name, err) + } + bc.Config = cfg + return nil + } + // This is reached when the for loop iterates over all entries, but didn't + // return. This means we had a loadBalancingConfig slice but did not + // encounter a registered policy. The config is considered invalid in this + // case. + return fmt.Errorf("invalid loadBalancingConfig: no supported policies found in %v", names) +} + +// MethodConfig defines the configuration recommended by the service providers for a +// particular method. +type MethodConfig struct { + // WaitForReady indicates whether RPCs sent to this method should wait until + // the connection is ready by default (!failfast). The value specified via the + // gRPC client API will override the value set here. + WaitForReady *bool + // Timeout is the default timeout for RPCs sent to this method. The actual + // deadline used will be the minimum of the value specified here and the value + // set by the application via the gRPC client API. If either one is not set, + // then the other will be used. If neither is set, then the RPC has no deadline. + Timeout *time.Duration + // MaxReqSize is the maximum allowed payload size for an individual request in a + // stream (client->server) in bytes. The size which is measured is the serialized + // payload after per-message compression (but before stream compression) in bytes. + // The actual value used is the minimum of the value specified here and the value set + // by the application via the gRPC client API. If either one is not set, then the other + // will be used. If neither is set, then the built-in default is used. + MaxReqSize *int + // MaxRespSize is the maximum allowed payload size for an individual response in a + // stream (server->client) in bytes. + MaxRespSize *int + // RetryPolicy configures retry options for the method. + RetryPolicy *RetryPolicy +} + +// RetryPolicy defines the go-native version of the retry policy defined by the +// service config here: +// https://github.com/grpc/proposal/blob/master/A6-client-retries.md#integration-with-service-config +type RetryPolicy struct { + // MaxAttempts is the maximum number of attempts, including the original RPC. + // + // This field is required and must be two or greater. + MaxAttempts int + + // Exponential backoff parameters. The initial retry attempt will occur at + // random(0, initialBackoff). In general, the nth attempt will occur at + // random(0, + // min(initialBackoff*backoffMultiplier**(n-1), maxBackoff)). + // + // These fields are required and must be greater than zero. + InitialBackoff time.Duration + MaxBackoff time.Duration + BackoffMultiplier float64 + + // The set of status codes which may be retried. + // + // Status codes are specified as strings, e.g., "UNAVAILABLE". + // + // This field is required and must be non-empty. + // Note: a set is used to store this for easy lookup. + RetryableStatusCodes map[codes.Code]bool +} diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go new file mode 100644 index 0000000000..3a725685db --- /dev/null +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -0,0 +1,182 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package serviceconfig + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + externalserviceconfig "google.golang.org/grpc/serviceconfig" +) + +type testBalancerConfigType struct { + externalserviceconfig.LoadBalancingConfig `json:"-"` + + Check bool `json:"check"` +} + +var testBalancerConfig = testBalancerConfigType{Check: true} + +const ( + testBalancerBuilderName = "test-bb" + testBalancerBuilderNotParserName = "test-bb-not-parser" + + testBalancerConfigJSON = `{"check":true}` +) + +type testBalancerBuilder struct { + balancer.Builder +} + +func (testBalancerBuilder) ParseConfig(js json.RawMessage) (externalserviceconfig.LoadBalancingConfig, error) { + if string(js) != testBalancerConfigJSON { + return nil, fmt.Errorf("unexpected config json") + } + return testBalancerConfig, nil +} + +func (testBalancerBuilder) Name() string { + return testBalancerBuilderName +} + +type testBalancerBuilderNotParser struct { + balancer.Builder +} + +func (testBalancerBuilderNotParser) Name() string { + return testBalancerBuilderNotParserName +} + +func init() { + balancer.Register(testBalancerBuilder{}) + balancer.Register(testBalancerBuilderNotParser{}) +} + +func TestBalancerConfigUnmarshalJSON(t *testing.T) { + tests := []struct { + name string + json string + want BalancerConfig + wantErr bool + }{ + { + name: "empty json", + json: "", + wantErr: true, + }, + { + // The config should be a slice of maps, but each map should have + // exactly one entry. + name: "more than one entry for a map", + json: `[{"balancer1":"1","balancer2":"2"}]`, + wantErr: true, + }, + { + name: "no balancer registered", + json: `[{"balancer1":"1"},{"balancer2":"2"}]`, + wantErr: true, + }, + { + name: "OK", + json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantErr: false, + }, + { + name: "first balancer not registered", + json: fmt.Sprintf(`[{"balancer1":"1"},{%q: %v}]`, testBalancerBuilderName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantErr: false, + }, + { + name: "balancer registered but builder not parser", + json: fmt.Sprintf("[{%q: %v}]", testBalancerBuilderNotParserName, testBalancerConfigJSON), + want: BalancerConfig{ + Name: testBalancerBuilderNotParserName, + Config: nil, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var bc BalancerConfig + if err := bc.UnmarshalJSON([]byte(tt.json)); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalJSON() error = %v, wantErr %v", err, tt.wantErr) + } + if !cmp.Equal(bc, tt.want) { + t.Errorf("diff: %v", cmp.Diff(bc, tt.want)) + } + }) + } +} + +func TestBalancerConfigMarshalJSON(t *testing.T) { + tests := []struct { + name string + bc BalancerConfig + wantJSON string + }{ + { + name: "OK", + bc: BalancerConfig{ + Name: testBalancerBuilderName, + Config: testBalancerConfig, + }, + wantJSON: `[{"test-bb": {"check":true}}]`, + }, + { + name: "OK config is nil", + bc: BalancerConfig{ + Name: testBalancerBuilderNotParserName, + Config: nil, // nil should be marshalled to an empty config "{}". + }, + wantJSON: `[{"test-bb-not-parser": {}}]`, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b, err := tt.bc.MarshalJSON() + if err != nil { + t.Fatalf("failed to marshal: %v", err) + } + + if str := string(b); str != tt.wantJSON { + t.Fatalf("got str %q, want %q", str, tt.wantJSON) + } + + var bc BalancerConfig + if err := bc.UnmarshalJSON(b); err != nil { + t.Errorf("failed to unmarshal: %v", err) + } + if !cmp.Equal(bc, tt.bc) { + t.Errorf("diff: %v", cmp.Diff(bc, tt.bc)) + } + }) + } +} diff --git a/xds/utils/wrr/edf.go b/xds/utils/wrr/edf.go new file mode 100644 index 0000000000..b4fb3f9d3b --- /dev/null +++ b/xds/utils/wrr/edf.go @@ -0,0 +1,92 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wrr + +import ( + "container/heap" + "sync" +) + +// edfWrr is a struct for EDF weighted round robin implementation. +type edfWrr struct { + lock sync.Mutex + items edfPriorityQueue + currentOrderOffset uint64 + currentTime float64 +} + +// NewEDF creates Earliest Deadline First (EDF) +// (https://en.wikipedia.org/wiki/Earliest_deadline_first_scheduling) implementation for weighted round robin. +// Each pick from the schedule has the earliest deadline entry selected. Entries have deadlines set +// at current time + 1 / weight, providing weighted round robin behavior with O(log n) pick time. +func NewEDF() WRR { + return &edfWrr{} +} + +// edfEntry is an internal wrapper for item that also stores weight and relative position in the queue. +type edfEntry struct { + deadline float64 + weight int64 + orderOffset uint64 + item interface{} +} + +// edfPriorityQueue is a heap.Interface implementation for edfEntry elements. +type edfPriorityQueue []*edfEntry + +func (pq edfPriorityQueue) Len() int { return len(pq) } +func (pq edfPriorityQueue) Less(i, j int) bool { + return pq[i].deadline < pq[j].deadline || pq[i].deadline == pq[j].deadline && pq[i].orderOffset < pq[j].orderOffset +} +func (pq edfPriorityQueue) Swap(i, j int) { pq[i], pq[j] = pq[j], pq[i] } + +func (pq *edfPriorityQueue) Push(x interface{}) { + *pq = append(*pq, x.(*edfEntry)) +} + +func (pq *edfPriorityQueue) Pop() interface{} { + old := *pq + *pq = old[0 : len(old)-1] + return old[len(old)-1] +} + +func (edf *edfWrr) Add(item interface{}, weight int64) { + edf.lock.Lock() + defer edf.lock.Unlock() + entry := edfEntry{ + deadline: edf.currentTime + 1.0/float64(weight), + weight: weight, + item: item, + orderOffset: edf.currentOrderOffset, + } + edf.currentOrderOffset++ + heap.Push(&edf.items, &entry) +} + +func (edf *edfWrr) Next() interface{} { + edf.lock.Lock() + defer edf.lock.Unlock() + if len(edf.items) == 0 { + return nil + } + item := edf.items[0] + edf.currentTime = item.deadline + item.deadline = edf.currentTime + 1.0/float64(item.weight) + heap.Fix(&edf.items, 0) + return item.item +} diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go new file mode 100644 index 0000000000..61fbf4a5e7 --- /dev/null +++ b/xds/utils/wrr/random.go @@ -0,0 +1,79 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package wrr + +import ( + "fmt" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" +) + +// weightedItem is a wrapped weighted item that is used to implement weighted random algorithm. +type weightedItem struct { + Item interface{} + Weight int64 +} + +func (w *weightedItem) String() string { + return fmt.Sprint(*w) +} + +// randomWRR is a struct that contains weighted items implement weighted random algorithm. +type randomWRR struct { + mu sync.RWMutex + items []*weightedItem + sumOfWeights int64 +} + +// NewRandom creates a new WRR with random. +func NewRandom() WRR { + return &randomWRR{} +} + +var grpcrandInt63n = grpcrand.Int63n + +func (rw *randomWRR) Next() (item interface{}) { + rw.mu.RLock() + defer rw.mu.RUnlock() + if rw.sumOfWeights == 0 { + return nil + } + // Random number in [0, sum). + randomWeight := grpcrandInt63n(rw.sumOfWeights) + for _, item := range rw.items { + randomWeight = randomWeight - item.Weight + if randomWeight < 0 { + return item.Item + } + } + + return rw.items[len(rw.items)-1].Item +} + +func (rw *randomWRR) Add(item interface{}, weight int64) { + rw.mu.Lock() + defer rw.mu.Unlock() + rItem := &weightedItem{Item: item, Weight: weight} + rw.items = append(rw.items, rItem) + rw.sumOfWeights += weight +} + +func (rw *randomWRR) String() string { + return fmt.Sprint(rw.items) +} diff --git a/xds/utils/wrr/wrr.go b/xds/utils/wrr/wrr.go new file mode 100644 index 0000000000..d46bfad86e --- /dev/null +++ b/xds/utils/wrr/wrr.go @@ -0,0 +1,32 @@ +/* + * + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package wrr contains the interface and common implementations of wrr +// algorithms. +package wrr + +// WRR defines an interface that implements weighted round robin. +type WRR interface { + // Add adds an item with weight to the WRR set. + // + // Add and Next need to be thread safe. + Add(item interface{}, weight int64) + // Next returns the next picked item. + // + // Add and Next need to be thread safe. + Next() interface{} +} diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go new file mode 100644 index 0000000000..e8b492774d --- /dev/null +++ b/xds/xds_handshake_cluster.go @@ -0,0 +1,40 @@ +/* + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package internal + +import ( + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" +) + +// handshakeClusterNameKey is the type used as the key to store cluster name in +// the Attributes field of resolver.Address. +type handshakeClusterNameKey struct{} + +// SetXDSHandshakeClusterName returns a copy of addr in which the Attributes field +// is updated with the cluster name. +func SetXDSHandshakeClusterName(addr resolver.Address, clusterName string) resolver.Address { + addr.Attributes = addr.Attributes.WithValue(handshakeClusterNameKey{}, clusterName) + return addr +} + +// GetXDSHandshakeClusterName returns cluster name stored in attr. +func GetXDSHandshakeClusterName(attr *attributes.Attributes) (string, bool) { + v := attr.Value(handshakeClusterNameKey{}) + name, ok := v.(string) + return name, ok +} From 193d021a22f5bf5840414784bc2bbdc2a139991d Mon Sep 17 00:00:00 2001 From: zhshw Date: Tue, 15 Mar 2022 08:18:35 +0800 Subject: [PATCH 02/19] xds server from grpc --- xds/internal.go | 2 +- xds/server.go | 399 +++++++++++++++++++++++++++++ xds/server/conn_wrapper.go | 165 ++++++++++++ xds/server/listener_wrapper.go | 442 +++++++++++++++++++++++++++++++++ xds/server/rds_handler.go | 133 ++++++++++ xds/server_options.go | 76 ++++++ xds/utils/transport/conn.go | 21 ++ xds/xds_handshake_cluster.go | 2 +- 8 files changed, 1238 insertions(+), 2 deletions(-) create mode 100644 xds/server.go create mode 100644 xds/server/conn_wrapper.go create mode 100644 xds/server/listener_wrapper.go create mode 100644 xds/server/rds_handler.go create mode 100644 xds/server_options.go create mode 100644 xds/utils/transport/conn.go diff --git a/xds/internal.go b/xds/internal.go index 1b596bf357..776b948bb6 100644 --- a/xds/internal.go +++ b/xds/internal.go @@ -18,7 +18,7 @@ // Package internal contains gRPC-internal code, to avoid polluting // the godoc of the top-level grpc package. It must not import any grpc // symbols to avoid circular dependencies. -package internal +package xds import ( "context" diff --git a/xds/server.go b/xds/server.go new file mode 100644 index 0000000000..caae913088 --- /dev/null +++ b/xds/server.go @@ -0,0 +1,399 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/server" + "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const serverPrefix = "[xds-server %p] " + +var ( + // These new functions will be overridden in unit tests. + newXDSClient = func() (client.XDSClient, error) { + return client.New() + } + newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { + return grpc.NewServer(opts...) + } + + grpcGetServerCreds = GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) + drainServerTransports = DrainServerTransports.(func(*grpc.Server, string)) + logger = grpclog.Component("xds") +) + +func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(serverPrefix, p)) +} + +// grpcServer contains methods from grpc.Server which are used by the +// GRPCServer type here. This is useful for overriding in unit tests. +type grpcServer interface { + RegisterService(*grpc.ServiceDesc, interface{}) + Serve(net.Listener) error + Stop() + GracefulStop() + GetServiceInfo() map[string]grpc.ServiceInfo +} + +// GRPCServer wraps a gRPC server and provides server-side xDS functionality, by +// communication with a management server using xDS APIs. It implements the +// grpc.ServiceRegistrar interface and can be passed to service registration +// functions in IDL generated code. +type GRPCServer struct { + gs grpcServer + quit *grpcsync.Event + logger *internalgrpclog.PrefixLogger + xdsCredsInUse bool + opts *serverOptions + + // clientMu is used only in initXDSClient(), which is called at the + // beginning of Serve(), where we have to decide if we have to create a + // client or use an existing one. + clientMu sync.Mutex + xdsC client.XDSClient +} + +// NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts. +// The underlying gRPC server has no service registered and has not started to +// accept requests yet. +func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer { + newOpts := []grpc.ServerOption{ + grpc.ChainUnaryInterceptor(xdsUnaryInterceptor), + grpc.ChainStreamInterceptor(xdsStreamInterceptor), + } + newOpts = append(newOpts, opts...) + s := &GRPCServer{ + gs: newGRPCServer(newOpts...), + quit: grpcsync.NewEvent(), + opts: handleServerOptions(opts), + } + s.logger = prefixLogger(s) + s.logger.Infof("Created xds.GRPCServer") + + // We type assert our underlying gRPC server to the real grpc.Server here + // before trying to retrieve the configured credentials. This approach + // avoids performing the same type assertion in the grpc package which + // provides the implementation for internal.GetServerCredentials, and allows + // us to use a fake gRPC server in tests. + if gs, ok := s.gs.(*grpc.Server); ok { + creds := grpcGetServerCreds(gs) + if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { + s.xdsCredsInUse = true + } + } + + s.logger.Infof("xDS credentials in use: %v", s.xdsCredsInUse) + return s +} + +// handleServerOptions iterates through the list of server options passed in by +// the user, and handles the xDS server specific options. +func handleServerOptions(opts []grpc.ServerOption) *serverOptions { + so := &serverOptions{} + for _, opt := range opts { + if o, ok := opt.(*serverOption); ok { + o.apply(so) + } + } + return so +} + +// RegisterService registers a service and its implementation to the underlying +// gRPC server. It is called from the IDL generated code. This must be called +// before invoking Serve. +func (s *GRPCServer) RegisterService(sd *grpc.ServiceDesc, ss interface{}) { + s.gs.RegisterService(sd, ss) +} + +// GetServiceInfo returns a map from service names to ServiceInfo. +// Service names include the package names, in the form of .. +func (s *GRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { + return s.gs.GetServiceInfo() +} + +// initXDSClient creates a new xdsClient if there is no existing one available. +func (s *GRPCServer) initXDSClient() error { + s.clientMu.Lock() + defer s.clientMu.Unlock() + + if s.xdsC != nil { + return nil + } + + newXDSClient := newXDSClient + if s.opts.bootstrapContents != nil { + newXDSClient = func() (client.XDSClient, error) { + return client.NewClientWithBootstrapContents(s.opts.bootstrapContents) + } + } + client, err := newXDSClient() + if err != nil { + return fmt.Errorf("xds: failed to create xds-client: %v", err) + } + s.xdsC = client + s.logger.Infof("Created an xdsClient") + return nil +} + +// Serve gets the underlying gRPC server to accept incoming connections on the +// listener lis, which is expected to be listening on a TCP port. +// +// A connection to the management server, to receive xDS configuration, is +// initiated here. +// +// Serve will return a non-nil error unless Stop or GracefulStop is called. +func (s *GRPCServer) Serve(lis net.Listener) error { + s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String()) + if _, ok := lis.Addr().(*net.TCPAddr); !ok { + return fmt.Errorf("xds: GRPCServer expects listener to return a net.TCPAddr. Got %T", lis.Addr()) + } + + // If this is the first time Serve() is being called, we need to initialize + // our xdsClient. If not, we can use the existing one. + if err := s.initXDSClient(); err != nil { + return err + } + cfg := s.xdsC.BootstrapConfig() + if cfg == nil { + return errors.New("bootstrap configuration is empty") + } + + // If xds credentials were specified by the user, but bootstrap configs do + // not contain any certificate provider configuration, it is better to fail + // right now rather than failing when attempting to create certificate + // providers after receiving an LDS response with security configuration. + if s.xdsCredsInUse { + if len(cfg.CertProviderConfigs) == 0 { + return errors.New("xds: certificate_providers config missing in bootstrap file") + } + } + + // The server listener resource name template from the bootstrap + // configuration contains a template for the name of the Listener resource + // to subscribe to for a gRPC server. If the token `%s` is present in the + // string, it will be replaced with the server's listening "IP:port" (e.g., + // "0.0.0.0:8080", "[::]:8080"). The absence of a template will be treated + // as an error since we do not have any default value for this. + if cfg.ServerListenerResourceNameTemplate == "" { + return errors.New("missing server_listener_resource_name_template in the bootstrap configuration") + } + name := bootstrap.PopulateResourceTemplate(cfg.ServerListenerResourceNameTemplate, lis.Addr().String()) + + modeUpdateCh := buffer.NewUnbounded() + go func() { + s.handleServingModeChanges(modeUpdateCh) + }() + + // Create a listenerWrapper which handles all functionality required by + // this particular instance of Serve(). + lw, goodUpdateCh := server.NewListenerWrapper(server.ListenerWrapperParams{ + Listener: lis, + ListenerResourceName: name, + XDSCredsInUse: s.xdsCredsInUse, + XDSClient: s.xdsC, + ModeCallback: func(addr net.Addr, mode connectivity.ServingMode, err error) { + modeUpdateCh.Put(&modeChangeArgs{ + addr: addr, + mode: mode, + err: err, + }) + }, + DrainCallback: func(addr net.Addr) { + if gs, ok := s.gs.(*grpc.Server); ok { + drainServerTransports(gs, addr.String()) + } + }, + }) + + // Block until a good LDS response is received or the server is stopped. + select { + case <-s.quit.Done(): + // Since the listener has not yet been handed over to gs.Serve(), we + // need to explicitly close the listener. Cancellation of the xDS watch + // is handled by the listenerWrapper. + lw.Close() + return nil + case <-goodUpdateCh: + } + return s.gs.Serve(lw) +} + +// modeChangeArgs wraps argument required for invoking mode change callback. +type modeChangeArgs struct { + addr net.Addr + mode connectivity.ServingMode + err error +} + +// handleServingModeChanges runs as a separate goroutine, spawned from Serve(). +// It reads a channel on to which mode change arguments are pushed, and in turn +// invokes the user registered callback. It also calls an internal method on the +// underlying grpc.Server to gracefully close existing connections, if the +// listener moved to a "not-serving" mode. +func (s *GRPCServer) handleServingModeChanges(updateCh *buffer.Unbounded) { + for { + select { + case <-s.quit.Done(): + return + case u := <-updateCh.Get(): + updateCh.Load() + args := u.(*modeChangeArgs) + if args.mode == connectivity.ServingModeNotServing { + // We type assert our underlying gRPC server to the real + // grpc.Server here before trying to initiate the drain + // operation. This approach avoids performing the same type + // assertion in the grpc package which provides the + // implementation for internal.GetServerCredentials, and allows + // us to use a fake gRPC server in tests. + if gs, ok := s.gs.(*grpc.Server); ok { + drainServerTransports(gs, args.addr.String()) + } + } + if s.opts.modeCallback != nil { + s.opts.modeCallback(args.addr, ServingModeChangeArgs{ + Mode: args.mode, + Err: args.err, + }) + } + } + } +} + +// Stop stops the underlying gRPC server. It immediately closes all open +// connections. It cancels all active RPCs on the server side and the +// corresponding pending RPCs on the client side will get notified by connection +// errors. +func (s *GRPCServer) Stop() { + s.quit.Fire() + s.gs.Stop() + if s.xdsC != nil { + s.xdsC.Close() + } +} + +// GracefulStop stops the underlying gRPC server gracefully. It stops the server +// from accepting new connections and RPCs and blocks until all the pending RPCs +// are finished. +func (s *GRPCServer) GracefulStop() { + s.quit.Fire() + s.gs.GracefulStop() + if s.xdsC != nil { + s.xdsC.Close() + } +} + +// routeAndProcess routes the incoming RPC to a configured route in the route +// table and also processes the RPC by running the incoming RPC through any HTTP +// Filters configured. +func routeAndProcess(ctx context.Context) error { + conn := transport.GetConnection(ctx) + cw, ok := conn.(interface { + VirtualHosts() []resource.VirtualHostWithInterceptors + }) + if !ok { + return errors.New("missing virtual hosts in incoming context") + } + mn, ok := grpc.Method(ctx) + if !ok { + return errors.New("missing method name in incoming context") + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return errors.New("missing metadata in incoming context") + } + // A41 added logic to the core grpc implementation to guarantee that once + // the RPC gets to this point, there will be a single, unambiguous authority + // present in the header map. + authority := md.Get(":authority") + vh := resource.FindBestMatchingVirtualHostServer(authority[0], cw.VirtualHosts()) + if vh == nil { + return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Virtual Host") + } + + var rwi *resource.RouteWithInterceptors + rpcInfo := iresolver.RPCInfo{ + Context: ctx, + Method: mn, + } + for _, r := range vh.Routes { + if r.M.Match(rpcInfo) { + // "NonForwardingAction is expected for all Routes used on server-side; a route with an inappropriate action causes + // RPCs matching that route to fail with UNAVAILABLE." - A36 + if r.ActionType != resource.RouteActionNonForwardingAction { + return status.Error(codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding") + } + rwi = &r + break + } + } + if rwi == nil { + return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Route") + } + for _, interceptor := range rwi.Interceptors { + if err := interceptor.AllowRPC(ctx); err != nil { + return status.Errorf(codes.PermissionDenied, "Incoming RPC is not allowed: %v", err) + } + } + return nil +} + +// xdsUnaryInterceptor is the unary interceptor added to the gRPC server to +// perform any xDS specific functionality on unary RPCs. +func xdsUnaryInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + if envconfig.XDSRBAC { + if err := routeAndProcess(ctx); err != nil { + return nil, err + } + } + return handler(ctx, req) +} + +// xdsStreamInterceptor is the stream interceptor added to the gRPC server to +// perform any xDS specific functionality on streaming RPCs. +func xdsStreamInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { + if envconfig.XDSRBAC { + if err := routeAndProcess(ss.Context()); err != nil { + return err + } + } + return handler(srv, ss) +} diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go new file mode 100644 index 0000000000..50ba42b19b --- /dev/null +++ b/xds/server/conn_wrapper.go @@ -0,0 +1,165 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "errors" + "fmt" + "net" + "sync" + "time" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" + "google.golang.org/grpc/credentials/tls/certprovider" +) + +// connWrapper is a thin wrapper around a net.Conn returned by Accept(). It +// provides the following additional functionality: +// 1. A way to retrieve the configured deadline. This is required by the +// ServerHandshake() method of the xdsCredentials when it attempts to read +// key material from the certificate providers. +// 2. Implements the XDSHandshakeInfo() method used by the xdsCredentials to +// retrieve the configured certificate providers. +// 3. xDS filter_chain matching logic to select appropriate security +// configuration for the incoming connection. +type connWrapper struct { + net.Conn + + // The specific filter chain picked for handling this connection. + filterChain *resource.FilterChain + + // A reference fo the listenerWrapper on which this connection was accepted. + parent *listenerWrapper + + // The certificate providers created for this connection. + rootProvider, identityProvider certprovider.Provider + + // The connection deadline as configured by the grpc.Server on the rawConn + // that is returned by a call to Accept(). This is set to the connection + // timeout value configured by the user (or to a default value) before + // initiating the transport credential handshake, and set to zero after + // completing the HTTP2 handshake. + deadlineMu sync.Mutex + deadline time.Time + + // The virtual hosts with matchable routes and instantiated HTTP Filters per + // route. + virtualHosts []resource.VirtualHostWithInterceptors +} + +// VirtualHosts returns the virtual hosts to be used for server side routing. +func (c *connWrapper) VirtualHosts() []resource.VirtualHostWithInterceptors { + return c.virtualHosts +} + +// SetDeadline makes a copy of the passed in deadline and forwards the call to +// the underlying rawConn. +func (c *connWrapper) SetDeadline(t time.Time) error { + c.deadlineMu.Lock() + c.deadline = t + c.deadlineMu.Unlock() + return c.Conn.SetDeadline(t) +} + +// GetDeadline returns the configured deadline. This will be invoked by the +// ServerHandshake() method of the XdsCredentials, which needs a deadline to +// pass to the certificate provider. +func (c *connWrapper) GetDeadline() time.Time { + c.deadlineMu.Lock() + t := c.deadline + c.deadlineMu.Unlock() + return t +} + +// XDSHandshakeInfo returns a HandshakeInfo with appropriate security +// configuration for this connection. This method is invoked by the +// ServerHandshake() method of the XdsCredentials. +func (c *connWrapper) XDSHandshakeInfo() (*xdsinternal.HandshakeInfo, error) { + // Ideally this should never happen, since xdsCredentials are the only ones + // which will invoke this method at handshake time. But to be on the safe + // side, we avoid acting on the security configuration received from the + // control plane when the user has not configured the use of xDS + // credentials, by checking the value of this flag. + if !c.parent.xdsCredsInUse { + return nil, errors.New("user has not configured xDS credentials") + } + + if c.filterChain.SecurityCfg == nil { + // If the security config is empty, this means that the control plane + // did not provide any security configuration and therefore we should + // return an empty HandshakeInfo here so that the xdsCreds can use the + // configured fallback credentials. + return xdsinternal.NewHandshakeInfo(nil, nil), nil + } + + cpc := c.parent.xdsC.BootstrapConfig().CertProviderConfigs + // Identity provider name is mandatory on the server-side, and this is + // enforced when the resource is received at the XDSClient layer. + secCfg := c.filterChain.SecurityCfg + ip, err := buildProviderFunc(cpc, secCfg.IdentityInstanceName, secCfg.IdentityCertName, true, false) + if err != nil { + return nil, err + } + // Root provider name is optional and required only when doing mTLS. + var rp certprovider.Provider + if instance, cert := secCfg.RootInstanceName, secCfg.RootCertName; instance != "" { + rp, err = buildProviderFunc(cpc, instance, cert, false, true) + if err != nil { + return nil, err + } + } + c.identityProvider = ip + c.rootProvider = rp + + xdsHI := xdsinternal.NewHandshakeInfo(c.rootProvider, c.identityProvider) + xdsHI.SetRequireClientCert(secCfg.RequireClientCert) + return xdsHI, nil +} + +// Close closes the providers and the underlying connection. +func (c *connWrapper) Close() error { + if c.identityProvider != nil { + c.identityProvider.Close() + } + if c.rootProvider != nil { + c.rootProvider.Close() + } + return c.Conn.Close() +} + +func buildProviderFunc(configs map[string]*certprovider.BuildableConfig, instanceName, certName string, wantIdentity, wantRoot bool) (certprovider.Provider, error) { + cfg, ok := configs[instanceName] + if !ok { + return nil, fmt.Errorf("certificate provider instance %q not found in bootstrap file", instanceName) + } + provider, err := cfg.Build(certprovider.BuildOptions{ + CertName: certName, + WantIdentity: wantIdentity, + WantRoot: wantRoot, + }) + if err != nil { + // This error is not expected since the bootstrap process parses the + // config and makes sure that it is acceptable to the plugin. Still, it + // is possible that the plugin parses the config successfully, but its + // Build() method errors out. + return nil, fmt.Errorf("failed to get security plugin instance (%+v): %v", cfg, err) + } + return provider, nil +} diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go new file mode 100644 index 0000000000..971bc58cfe --- /dev/null +++ b/xds/server/listener_wrapper.go @@ -0,0 +1,442 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package server contains internal server-side functionality used by the public +// facing xds package. +package server + +import ( + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + "unsafe" + + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + internalbackoff "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/grpclog" +) + +var ( + logger = grpclog.Component("xds") + + // Backoff strategy for temporary errors received from Accept(). If this + // needs to be configurable, we can inject it through ListenerWrapperParams. + bs = internalbackoff.Exponential{Config: backoff.Config{ + BaseDelay: 5 * time.Millisecond, + Multiplier: 2.0, + MaxDelay: 1 * time.Second, + }} + backoffFunc = bs.Backoff +) + +// ServingModeCallback is the callback that users can register to get notified +// about the server's serving mode changes. The callback is invoked with the +// address of the listener and its new mode. The err parameter is set to a +// non-nil error if the server has transitioned into not-serving mode. +type ServingModeCallback func(addr net.Addr, mode connectivity.ServingMode, err error) + +// DrainCallback is the callback that an xDS-enabled server registers to get +// notified about updates to the Listener configuration. The server is expected +// to gracefully shutdown existing connections, thereby forcing clients to +// reconnect and have the new configuration applied to the newly created +// connections. +type DrainCallback func(addr net.Addr) + +func prefixLogger(p *listenerWrapper) *internalgrpclog.PrefixLogger { + return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf("[xds-server-listener %p] ", p)) +} + +// XDSClient wraps the methods on the XDSClient which are required by +// the listenerWrapper. +type XDSClient interface { + WatchListener(string, func(resource.ListenerUpdate, error)) func() + WatchRouteConfig(string, func(resource.RouteConfigUpdate, error)) func() + BootstrapConfig() *bootstrap.Config +} + +// ListenerWrapperParams wraps parameters required to create a listenerWrapper. +type ListenerWrapperParams struct { + // Listener is the net.Listener passed by the user that is to be wrapped. + Listener net.Listener + // ListenerResourceName is the xDS Listener resource to request. + ListenerResourceName string + // XDSCredsInUse specifies whether or not the user expressed interest to + // receive security configuration from the control plane. + XDSCredsInUse bool + // XDSClient provides the functionality from the XDSClient required here. + XDSClient XDSClient + // ModeCallback is the callback to invoke when the serving mode changes. + ModeCallback ServingModeCallback + // DrainCallback is the callback to invoke when the Listener gets a LDS + // update. + DrainCallback DrainCallback +} + +// NewListenerWrapper creates a new listenerWrapper with params. It returns a +// net.Listener and a channel which is written to, indicating that the former is +// ready to be passed to grpc.Serve(). +// +// Only TCP listeners are supported. +func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan struct{}) { + lw := &listenerWrapper{ + Listener: params.Listener, + name: params.ListenerResourceName, + xdsCredsInUse: params.XDSCredsInUse, + xdsC: params.XDSClient, + modeCallback: params.ModeCallback, + drainCallback: params.DrainCallback, + isUnspecifiedAddr: params.Listener.Addr().(*net.TCPAddr).IP.IsUnspecified(), + + closed: grpcsync.NewEvent(), + goodUpdate: grpcsync.NewEvent(), + ldsUpdateCh: make(chan ldsUpdateWithError, 1), + rdsUpdateCh: make(chan rdsHandlerUpdate, 1), + } + lw.logger = prefixLogger(lw) + + // Serve() verifies that Addr() returns a valid TCPAddr. So, it is safe to + // ignore the error from SplitHostPort(). + lisAddr := lw.Listener.Addr().String() + lw.addr, lw.port, _ = net.SplitHostPort(lisAddr) + + lw.rdsHandler = newRDSHandler(lw.xdsC, lw.rdsUpdateCh) + + cancelWatch := lw.xdsC.WatchListener(lw.name, lw.handleListenerUpdate) + lw.logger.Infof("Watch started on resource name %v", lw.name) + lw.cancelWatch = func() { + cancelWatch() + lw.logger.Infof("Watch cancelled on resource name %v", lw.name) + } + go lw.run() + return lw, lw.goodUpdate.Done() +} + +type ldsUpdateWithError struct { + update resource.ListenerUpdate + err error +} + +// listenerWrapper wraps the net.Listener associated with the listening address +// passed to Serve(). It also contains all other state associated with this +// particular invocation of Serve(). +type listenerWrapper struct { + net.Listener + logger *internalgrpclog.PrefixLogger + + name string + xdsCredsInUse bool + xdsC XDSClient + cancelWatch func() + modeCallback ServingModeCallback + drainCallback DrainCallback + + // Set to true if the listener is bound to the IP_ANY address (which is + // "0.0.0.0" for IPv4 and "::" for IPv6). + isUnspecifiedAddr bool + // Listening address and port. Used to validate the socket address in the + // Listener resource received from the control plane. + addr, port string + + // This is used to notify that a good update has been received and that + // Serve() can be invoked on the underlying gRPC server. Using an event + // instead of a vanilla channel simplifies the update handler as it need not + // keep track of whether the received update is the first one or not. + goodUpdate *grpcsync.Event + // A small race exists in the XDSClient code between the receipt of an xDS + // response and the user cancelling the associated watch. In this window, + // the registered callback may be invoked after the watch is canceled, and + // the user is expected to work around this. This event signifies that the + // listener is closed (and hence the watch is cancelled), and we drop any + // updates received in the callback if this event has fired. + closed *grpcsync.Event + + // mu guards access to the current serving mode and the filter chains. The + // reason for using an rw lock here is that these fields are read in + // Accept() for all incoming connections, but writes happen rarely (when we + // get a Listener resource update). + mu sync.RWMutex + // Current serving mode. + mode connectivity.ServingMode + // Filter chains received as part of the last good update. + filterChains *resource.FilterChainManager + + // rdsHandler is used for any dynamic RDS resources specified in a LDS + // update. + rdsHandler *rdsHandler + // rdsUpdates are the RDS resources received from the management + // server, keyed on the RouteName of the RDS resource. + rdsUpdates unsafe.Pointer // map[string]xdsclient.RouteConfigUpdate + // ldsUpdateCh is a channel for XDSClient LDS updates. + ldsUpdateCh chan ldsUpdateWithError + // rdsUpdateCh is a channel for XDSClient RDS updates. + rdsUpdateCh chan rdsHandlerUpdate +} + +// Accept blocks on an Accept() on the underlying listener, and wraps the +// returned net.connWrapper with the configured certificate providers. +func (l *listenerWrapper) Accept() (net.Conn, error) { + var retries int + for { + conn, err := l.Listener.Accept() + if err != nil { + // Temporary() method is implemented by certain error types returned + // from the net package, and it is useful for us to not shutdown the + // server in these conditions. The listen queue being full is one + // such case. + if ne, ok := err.(interface{ Temporary() bool }); !ok || !ne.Temporary() { + return nil, err + } + retries++ + timer := time.NewTimer(backoffFunc(retries)) + select { + case <-timer.C: + case <-l.closed.Done(): + timer.Stop() + // Continuing here will cause us to call Accept() again + // which will return a non-temporary error. + continue + } + continue + } + // Reset retries after a successful Accept(). + retries = 0 + + // Since the net.Conn represents an incoming connection, the source and + // destination address can be retrieved from the local address and + // remote address of the net.Conn respectively. + destAddr, ok1 := conn.LocalAddr().(*net.TCPAddr) + srcAddr, ok2 := conn.RemoteAddr().(*net.TCPAddr) + if !ok1 || !ok2 { + // If the incoming connection is not a TCP connection, which is + // really unexpected since we check whether the provided listener is + // a TCP listener in Serve(), we return an error which would cause + // us to stop serving. + return nil, fmt.Errorf("received connection with non-TCP address (local: %T, remote %T)", conn.LocalAddr(), conn.RemoteAddr()) + } + + l.mu.RLock() + if l.mode == connectivity.ServingModeNotServing { + // Close connections as soon as we accept them when we are in + // "not-serving" mode. Since we accept a net.Listener from the user + // in Serve(), we cannot close the listener when we move to + // "not-serving". Closing the connection immediately upon accepting + // is one of the other ways to implement the "not-serving" mode as + // outlined in gRFC A36. + l.mu.RUnlock() + conn.Close() + continue + } + fc, err := l.filterChains.Lookup(resource.FilterChainLookupParams{ + IsUnspecifiedListener: l.isUnspecifiedAddr, + DestAddr: destAddr.IP, + SourceAddr: srcAddr.IP, + SourcePort: srcAddr.Port, + }) + l.mu.RUnlock() + if err != nil { + // When a matching filter chain is not found, we close the + // connection right away, but do not return an error back to + // `grpc.Serve()` from where this Accept() was invoked. Returning an + // error to `grpc.Serve()` causes the server to shutdown. If we want + // to avoid the server from shutting down, we would need to return + // an error type which implements the `Temporary() bool` method, + // which is invoked by `grpc.Serve()` to see if the returned error + // represents a temporary condition. In the case of a temporary + // error, `grpc.Serve()` method sleeps for a small duration and + // therefore ends up blocking all connection attempts during that + // time frame, which is also not ideal for an error like this. + l.logger.Warningf("connection from %s to %s failed to find any matching filter chain", conn.RemoteAddr().String(), conn.LocalAddr().String()) + conn.Close() + continue + } + if !envconfig.XDSRBAC { + return &connWrapper{Conn: conn, filterChain: fc, parent: l}, nil + } + var rc resource.RouteConfigUpdate + if fc.InlineRouteConfig != nil { + rc = *fc.InlineRouteConfig + } else { + rcPtr := atomic.LoadPointer(&l.rdsUpdates) + rcuPtr := (*map[string]resource.RouteConfigUpdate)(rcPtr) + // This shouldn't happen, but this error protects against a panic. + if rcuPtr == nil { + return nil, errors.New("route configuration pointer is nil") + } + rcu := *rcuPtr + rc = rcu[fc.RouteConfigName] + } + // The filter chain will construct a usuable route table on each + // connection accept. This is done because preinstantiating every route + // table before it is needed for a connection would potentially lead to + // a lot of cpu time and memory allocated for route tables that will + // never be used. There was also a thought to cache this configuration, + // and reuse it for the next accepted connection. However, this would + // lead to a lot of code complexity (RDS Updates for a given route name + // can come it at any time), and connections aren't accepted too often, + // so this reinstantation of the Route Configuration is an acceptable + // tradeoff for simplicity. + vhswi, err := fc.ConstructUsableRouteConfiguration(rc) + if err != nil { + l.logger.Warningf("route configuration construction: %v", err) + conn.Close() + continue + } + return &connWrapper{Conn: conn, filterChain: fc, parent: l, virtualHosts: vhswi}, nil + } +} + +// Close closes the underlying listener. It also cancels the xDS watch +// registered in Serve() and closes any certificate provider instances created +// based on security configuration received in the LDS response. +func (l *listenerWrapper) Close() error { + l.closed.Fire() + l.Listener.Close() + if l.cancelWatch != nil { + l.cancelWatch() + } + l.rdsHandler.close() + return nil +} + +// run is a long running goroutine which handles all xds updates. LDS and RDS +// push updates onto a channel which is read and acted upon from this goroutine. +func (l *listenerWrapper) run() { + for { + select { + case <-l.closed.Done(): + return + case u := <-l.ldsUpdateCh: + l.handleLDSUpdate(u) + case u := <-l.rdsUpdateCh: + l.handleRDSUpdate(u) + } + } +} + +// handleLDSUpdate is the callback which handles LDS Updates. It writes the +// received update to the update channel, which is picked up by the run +// goroutine. +func (l *listenerWrapper) handleListenerUpdate(update resource.ListenerUpdate, err error) { + if l.closed.HasFired() { + l.logger.Warningf("Resource %q received update: %v with error: %v, after listener was closed", l.name, update, err) + return + } + // Remove any existing entry in ldsUpdateCh and replace with the new one, as the only update + // listener cares about is most recent update. + select { + case <-l.ldsUpdateCh: + default: + } + l.ldsUpdateCh <- ldsUpdateWithError{update: update, err: err} +} + +// handleRDSUpdate handles a full rds update from rds handler. On a successful +// update, the server will switch to ServingModeServing as the full +// configuration (both LDS and RDS) has been received. +func (l *listenerWrapper) handleRDSUpdate(update rdsHandlerUpdate) { + if l.closed.HasFired() { + l.logger.Warningf("RDS received update: %v with error: %v, after listener was closed", update.updates, update.err) + return + } + if update.err != nil { + l.logger.Warningf("Received error for rds names specified in resource %q: %+v", l.name, update.err) + if resource.ErrType(update.err) == resource.ErrorTypeResourceNotFound { + l.switchMode(nil, connectivity.ServingModeNotServing, update.err) + } + // For errors which are anything other than "resource-not-found", we + // continue to use the old configuration. + return + } + atomic.StorePointer(&l.rdsUpdates, unsafe.Pointer(&update.updates)) + + l.switchMode(l.filterChains, connectivity.ServingModeServing, nil) + l.goodUpdate.Fire() +} + +func (l *listenerWrapper) handleLDSUpdate(update ldsUpdateWithError) { + if update.err != nil { + l.logger.Warningf("Received error for resource %q: %+v", l.name, update.err) + if resource.ErrType(update.err) == resource.ErrorTypeResourceNotFound { + l.switchMode(nil, connectivity.ServingModeNotServing, update.err) + } + // For errors which are anything other than "resource-not-found", we + // continue to use the old configuration. + return + } + l.logger.Infof("Received update for resource %q: %+v", l.name, update.update) + + // Make sure that the socket address on the received Listener resource + // matches the address of the net.Listener passed to us by the user. This + // check is done here instead of at the XDSClient layer because of the + // following couple of reasons: + // - XDSClient cannot know the listening address of every listener in the + // system, and hence cannot perform this check. + // - this is a very context-dependent check and only the server has the + // appropriate context to perform this check. + // + // What this means is that the XDSClient has ACKed a resource which can push + // the server into a "not serving" mode. This is not ideal, but this is + // what we have decided to do. See gRPC A36 for more details. + ilc := update.update.InboundListenerCfg + if ilc.Address != l.addr || ilc.Port != l.port { + l.switchMode(nil, connectivity.ServingModeNotServing, fmt.Errorf("address (%s:%s) in Listener update does not match listening address: (%s:%s)", ilc.Address, ilc.Port, l.addr, l.port)) + return + } + + // "Updates to a Listener cause all older connections on that Listener to be + // gracefully shut down with a grace period of 10 minutes for long-lived + // RPC's, such that clients will reconnect and have the updated + // configuration apply." - A36 Note that this is not the same as moving the + // Server's state to ServingModeNotServing. That prevents new connections + // from being accepted, whereas here we simply want the clients to reconnect + // to get the updated configuration. + if envconfig.XDSRBAC { + if l.drainCallback != nil { + l.drainCallback(l.Listener.Addr()) + } + } + l.rdsHandler.updateRouteNamesToWatch(ilc.FilterChains.RouteConfigNames) + // If there are no dynamic RDS Configurations still needed to be received + // from the management server, this listener has all the configuration + // needed, and is ready to serve. + if len(ilc.FilterChains.RouteConfigNames) == 0 { + l.switchMode(ilc.FilterChains, connectivity.ServingModeServing, nil) + l.goodUpdate.Fire() + } +} + +func (l *listenerWrapper) switchMode(fcs *resource.FilterChainManager, newMode connectivity.ServingMode, err error) { + l.mu.Lock() + defer l.mu.Unlock() + + l.filterChains = fcs + l.mode = newMode + if l.modeCallback != nil { + l.modeCallback(l.Listener.Addr(), newMode, err) + } + l.logger.Warningf("Listener %q entering mode: %q due to error: %v", l.Addr(), newMode, err) +} diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go new file mode 100644 index 0000000000..552ac0e40d --- /dev/null +++ b/xds/server/rds_handler.go @@ -0,0 +1,133 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package server + +import ( + "sync" + + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// rdsHandlerUpdate wraps the full RouteConfigUpdate that are dynamically +// queried for a given server side listener. +type rdsHandlerUpdate struct { + updates map[string]resource.RouteConfigUpdate + err error +} + +// rdsHandler handles any RDS queries that need to be started for a given server +// side listeners Filter Chains (i.e. not inline). +type rdsHandler struct { + xdsC XDSClient + + mu sync.Mutex + updates map[string]resource.RouteConfigUpdate + cancels map[string]func() + + // For a rdsHandler update, the only update wrapped listener cares about is + // most recent one, so this channel will be opportunistically drained before + // sending any new updates. + updateChannel chan rdsHandlerUpdate +} + +// newRDSHandler creates a new rdsHandler to watch for RDS resources. +// listenerWrapper updates the list of route names to watch by calling +// updateRouteNamesToWatch() upon receipt of new Listener configuration. +func newRDSHandler(xdsC XDSClient, ch chan rdsHandlerUpdate) *rdsHandler { + return &rdsHandler{ + xdsC: xdsC, + updateChannel: ch, + updates: make(map[string]resource.RouteConfigUpdate), + cancels: make(map[string]func()), + } +} + +// updateRouteNamesToWatch handles a list of route names to watch for a given +// server side listener (if a filter chain specifies dynamic RDS configuration). +// This function handles all the logic with respect to any routes that may have +// been added or deleted as compared to what was previously present. +func (rh *rdsHandler) updateRouteNamesToWatch(routeNamesToWatch map[string]bool) { + rh.mu.Lock() + defer rh.mu.Unlock() + // Add and start watches for any routes for any new routes in + // routeNamesToWatch. + for routeName := range routeNamesToWatch { + if _, ok := rh.cancels[routeName]; !ok { + func(routeName string) { + rh.cancels[routeName] = rh.xdsC.WatchRouteConfig(routeName, func(update resource.RouteConfigUpdate, err error) { + rh.handleRouteUpdate(routeName, update, err) + }) + }(routeName) + } + } + + // Delete and cancel watches for any routes from persisted routeNamesToWatch + // that are no longer present. + for routeName := range rh.cancels { + if _, ok := routeNamesToWatch[routeName]; !ok { + rh.cancels[routeName]() + delete(rh.cancels, routeName) + delete(rh.updates, routeName) + } + } + + // If the full list (determined by length) of updates are now successfully + // updated, the listener is ready to be updated. + if len(rh.updates) == len(rh.cancels) && len(routeNamesToWatch) != 0 { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{updates: rh.updates}) + } +} + +// handleRouteUpdate persists the route config for a given route name, and also +// sends an update to the Listener Wrapper on an error received or if the rds +// handler has a full collection of updates. +func (rh *rdsHandler) handleRouteUpdate(routeName string, update resource.RouteConfigUpdate, err error) { + if err != nil { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{err: err}) + return + } + rh.mu.Lock() + defer rh.mu.Unlock() + rh.updates[routeName] = update + + // If the full list (determined by length) of updates have successfully + // updated, the listener is ready to be updated. + if len(rh.updates) == len(rh.cancels) { + drainAndPush(rh.updateChannel, rdsHandlerUpdate{updates: rh.updates}) + } +} + +func drainAndPush(ch chan rdsHandlerUpdate, update rdsHandlerUpdate) { + select { + case <-ch: + default: + } + ch <- update +} + +// close() is meant to be called by wrapped listener when the wrapped listener +// is closed, and it cleans up resources by canceling all the active RDS +// watches. +func (rh *rdsHandler) close() { + rh.mu.Lock() + defer rh.mu.Unlock() + for _, cancel := range rh.cancels { + cancel() + } +} diff --git a/xds/server_options.go b/xds/server_options.go new file mode 100644 index 0000000000..1d46c3adb7 --- /dev/null +++ b/xds/server_options.go @@ -0,0 +1,76 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package xds + +import ( + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" +) + +type serverOptions struct { + modeCallback ServingModeCallbackFunc + bootstrapContents []byte +} + +type serverOption struct { + grpc.EmptyServerOption + apply func(*serverOptions) +} + +// ServingModeCallback returns a grpc.ServerOption which allows users to +// register a callback to get notified about serving mode changes. +func ServingModeCallback(cb ServingModeCallbackFunc) grpc.ServerOption { + return &serverOption{apply: func(o *serverOptions) { o.modeCallback = cb }} +} + +// ServingModeCallbackFunc is the callback that users can register to get +// notified about the server's serving mode changes. The callback is invoked +// with the address of the listener and its new mode. +// +// Users must not perform any blocking operations in this callback. +type ServingModeCallbackFunc func(addr net.Addr, args ServingModeChangeArgs) + +// ServingModeChangeArgs wraps the arguments passed to the serving mode callback +// function. +type ServingModeChangeArgs struct { + // Mode is the new serving mode of the server listener. + Mode connectivity.ServingMode + // Err is set to a non-nil error if the server has transitioned into + // not-serving mode. + Err error +} + +// BootstrapContentsForTesting returns a grpc.ServerOption which allows users +// to inject a bootstrap configuration used by only this server, instead of the +// global configuration from the environment variables. +// +// Testing Only +// +// This function should ONLY be used for testing and may not work with some +// other features, including the CSDS service. +// +// Experimental +// +// Notice: This API is EXPERIMENTAL and may be changed or removed in a +// later release. +func BootstrapContentsForTesting(contents []byte) grpc.ServerOption { + return &serverOption{apply: func(o *serverOptions) { o.bootstrapContents = contents }} +} diff --git a/xds/utils/transport/conn.go b/xds/utils/transport/conn.go new file mode 100644 index 0000000000..16caf9dd26 --- /dev/null +++ b/xds/utils/transport/conn.go @@ -0,0 +1,21 @@ +package transport + +import ( + "context" + "net" +) + +type connectionKey struct{} + +// GetConnection gets the connection from the context. +func GetConnection(ctx context.Context) net.Conn { + conn, _ := ctx.Value(connectionKey{}).(net.Conn) + return conn +} + +// SetConnection adds the connection to the context to be able to get +// information about the destination ip and port for an incoming RPC. This also +// allows any unary or streaming interceptors to see the connection. +func setConnection(ctx context.Context, conn net.Conn) context.Context { + return context.WithValue(ctx, connectionKey{}, conn) +} diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index e8b492774d..b28f376b50 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package internal +package xds import ( "google.golang.org/grpc/attributes" From 6dd55e7f528babd139d9d4b344a2925d876f5db4 Mon Sep 17 00:00:00 2001 From: zhshw Date: Tue, 15 Mar 2022 09:18:20 +0800 Subject: [PATCH 03/19] xds csds from grpc --- xds/csds/csds.go | 216 ++++++++++++++++++ .../transport/networktype/networktype.go | 46 ++++ 2 files changed, 262 insertions(+) create mode 100644 xds/csds/csds.go create mode 100644 xds/utils/transport/networktype/networktype.go diff --git a/xds/csds/csds.go b/xds/csds/csds.go new file mode 100644 index 0000000000..6bbb8e981f --- /dev/null +++ b/xds/csds/csds.go @@ -0,0 +1,216 @@ +/* + * + * Copyright 2021 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package csds implements features to dump the status (xDS responses) the +// xds_client is using. +// +// Notice: This package is EXPERIMENTAL and may be changed or removed in a later +// release. +package csds + +import ( + "context" + "io" + + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" + + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" // Register v2 xds_client. + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" // Register v3 xds_client. +) + +var ( + logger = grpclog.Component("xds") + newXDSClient = func() client.XDSClient { + c, err := client.New() + if err != nil { + logger.Warningf("failed to create xds client: %v", err) + return nil + } + return c + } +) + +const ( + listenerTypeURL = "envoy.config.listener.v3.Listener" + routeConfigTypeURL = "envoy.config.route.v3.RouteConfiguration" + clusterTypeURL = "envoy.config.cluster.v3.Cluster" + endpointsTypeURL = "envoy.config.endpoint.v3.ClusterLoadAssignment" +) + +// ClientStatusDiscoveryServer implementations interface ClientStatusDiscoveryServiceServer. +type ClientStatusDiscoveryServer struct { + // xdsClient will always be the same in practice. But we keep a copy in each + // server instance for testing. + xdsClient client.XDSClient +} + +// NewClientStatusDiscoveryServer returns an implementation of the CSDS server that can be +// registered on a gRPC server. +func NewClientStatusDiscoveryServer() (*ClientStatusDiscoveryServer, error) { + return &ClientStatusDiscoveryServer{xdsClient: newXDSClient()}, nil +} + +// StreamClientStatus implementations interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) StreamClientStatus(stream v3statusgrpc.ClientStatusDiscoveryService_StreamClientStatusServer) error { + for { + req, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return err + } + resp, err := s.buildClientStatusRespForReq(req) + if err != nil { + return err + } + if err := stream.Send(resp); err != nil { + return err + } + } +} + +// FetchClientStatus implementations interface ClientStatusDiscoveryServiceServer. +func (s *ClientStatusDiscoveryServer) FetchClientStatus(_ context.Context, req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + return s.buildClientStatusRespForReq(req) +} + +// buildClientStatusRespForReq fetches the status from the client, and returns +// the response to be sent back to xdsclient. +// +// If it returns an error, the error is a status error. +func (s *ClientStatusDiscoveryServer) buildClientStatusRespForReq(req *v3statuspb.ClientStatusRequest) (*v3statuspb.ClientStatusResponse, error) { + if s.xdsClient == nil { + return &v3statuspb.ClientStatusResponse{}, nil + } + // Field NodeMatchers is unsupported, by design + // https://github.com/grpc/proposal/blob/master/A40-csds-support.md#detail-node-matching. + if len(req.NodeMatchers) != 0 { + return nil, status.Errorf(codes.InvalidArgument, "node_matchers are not supported, request contains node_matchers: %v", req.NodeMatchers) + } + + lds := dumpToGenericXdsConfig(listenerTypeURL, s.xdsClient.DumpLDS) + rds := dumpToGenericXdsConfig(routeConfigTypeURL, s.xdsClient.DumpRDS) + cds := dumpToGenericXdsConfig(clusterTypeURL, s.xdsClient.DumpCDS) + eds := dumpToGenericXdsConfig(endpointsTypeURL, s.xdsClient.DumpEDS) + configs := make([]*v3statuspb.ClientConfig_GenericXdsConfig, 0, len(lds)+len(rds)+len(cds)+len(eds)) + configs = append(configs, lds...) + configs = append(configs, rds...) + configs = append(configs, cds...) + configs = append(configs, eds...) + + ret := &v3statuspb.ClientStatusResponse{ + Config: []*v3statuspb.ClientConfig{ + { + Node: nodeProtoToV3(s.xdsClient.BootstrapConfig().XDSServer.NodeProto), + GenericXdsConfigs: configs, + }, + }, + } + return ret, nil +} + +// Close cleans up the resources. +func (s *ClientStatusDiscoveryServer) Close() { + if s.xdsClient != nil { + s.xdsClient.Close() + } +} + +// nodeProtoToV3 converts the given proto into a v3.Node. n is from bootstrap +// config, it can be either v2.Node or v3.Node. +// +// If n is already a v3.Node, return it. +// If n is v2.Node, marshal and unmarshal it to v3. +// Otherwise, return nil. +// +// The default case (not v2 or v3) is nil, instead of error, because the +// resources in the response are more important than the node. The worst case is +// that the user will receive no Node info, but will still get resources. +func nodeProtoToV3(n proto.Message) *v3corepb.Node { + var node *v3corepb.Node + switch nn := n.(type) { + case *v3corepb.Node: + node = nn + case *v2corepb.Node: + v2, err := proto.Marshal(nn) + if err != nil { + logger.Warningf("Failed to marshal node (%v): %v", n, err) + break + } + node = new(v3corepb.Node) + if err := proto.Unmarshal(v2, node); err != nil { + logger.Warningf("Failed to unmarshal node (%v): %v", v2, err) + } + default: + logger.Warningf("node from bootstrap is %#v, only v2.Node and v3.Node are supported", nn) + } + return node +} + +func dumpToGenericXdsConfig(typeURL string, dumpF func() map[string]resource.UpdateWithMD) []*v3statuspb.ClientConfig_GenericXdsConfig { + dump := dumpF() + ret := make([]*v3statuspb.ClientConfig_GenericXdsConfig, 0, len(dump)) + for name, d := range dump { + config := &v3statuspb.ClientConfig_GenericXdsConfig{ + TypeUrl: typeURL, + Name: name, + VersionInfo: d.MD.Version, + XdsConfig: d.Raw, + LastUpdated: timestamppb.New(d.MD.Timestamp), + ClientStatus: serviceStatusToProto(d.MD.Status), + } + if errState := d.MD.ErrState; errState != nil { + config.ErrorState = &v3adminpb.UpdateFailureState{ + LastUpdateAttempt: timestamppb.New(errState.Timestamp), + Details: errState.Err.Error(), + VersionInfo: errState.Version, + } + } + ret = append(ret, config) + } + return ret +} + +func serviceStatusToProto(serviceStatus resource.ServiceStatus) v3adminpb.ClientResourceStatus { + switch serviceStatus { + case resource.ServiceStatusUnknown: + return v3adminpb.ClientResourceStatus_UNKNOWN + case resource.ServiceStatusRequested: + return v3adminpb.ClientResourceStatus_REQUESTED + case resource.ServiceStatusNotExist: + return v3adminpb.ClientResourceStatus_DOES_NOT_EXIST + case resource.ServiceStatusACKed: + return v3adminpb.ClientResourceStatus_ACKED + case resource.ServiceStatusNACKed: + return v3adminpb.ClientResourceStatus_NACKED + default: + return v3adminpb.ClientResourceStatus_UNKNOWN + } +} diff --git a/xds/utils/transport/networktype/networktype.go b/xds/utils/transport/networktype/networktype.go new file mode 100644 index 0000000000..c11b527827 --- /dev/null +++ b/xds/utils/transport/networktype/networktype.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +// Package networktype declares the network type to be used in the default +// dialer. Attribute of a resolver.Address. +package networktype + +import ( + "google.golang.org/grpc/resolver" +) + +// keyType is the key to use for storing State in Attributes. +type keyType string + +const key = keyType("grpc.internal.transport.networktype") + +// Set returns a copy of the provided address with attributes containing networkType. +func Set(address resolver.Address, networkType string) resolver.Address { + address.Attributes = address.Attributes.WithValue(key, networkType) + return address +} + +// Get returns the network type in the resolver.Address and true, or "", false +// if not present. +func Get(address resolver.Address) (string, bool) { + v := address.Attributes.Value(key) + if v == nil { + return "", false + } + return v.(string), true +} From ad7a5303f04b233c5a656a2f48418bfc94dfbc07 Mon Sep 17 00:00:00 2001 From: zhshw Date: Tue, 15 Mar 2022 12:15:35 +0800 Subject: [PATCH 04/19] xds bootstrap from grpc --- xds/client/bootstrap/bootstrap.go | 2 +- xds/client/bootstrap/bootstrap_test.go | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 646a858e37..2b9c734ee7 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -27,7 +27,7 @@ import ( "io/ioutil" "strings" - "dubbo.apache.org/dubbo-go/v3/xds" + internal "dubbo.apache.org/dubbo-go/v3/xds" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 698d4bc0bb..37e4b99b44 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -25,13 +25,12 @@ import ( "os" "testing" + internal "dubbo.apache.org/dubbo-go/v3/xds" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "github.com/golang/protobuf/proto" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - - "dubbo.apache.org/dubbo-go/v3/xds" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" From 339db0885976f4284b9c92557864b1c5934efeb7 Mon Sep 17 00:00:00 2001 From: "lizhixin.lzx" Date: Wed, 16 Mar 2022 20:25:40 +0800 Subject: [PATCH 05/19] Fix: init test client --- test/xds/main.go | 60 ++++++++++ xds/client/bootstrap/bootstrap.go | 4 +- xds/client/bootstrap/bootstrap_test.go | 4 +- xds/client/client.go | 2 +- xds/{ => internal}/internal.go | 2 +- xds/server.go | 5 +- xds/utils/balancergroup/balancergroup.go | 2 +- xds/utils/grpclog/grpclog.go | 11 +- xds/utils/grpclog/prefixLogger.go | 3 - xds/utils/xds_cache/timeoutCache.go | 143 +++++++++++++++++++++++ 10 files changed, 219 insertions(+), 17 deletions(-) create mode 100644 test/xds/main.go rename xds/{ => internal}/internal.go (99%) create mode 100644 xds/utils/xds_cache/timeoutCache.go diff --git a/test/xds/main.go b/test/xds/main.go new file mode 100644 index 0000000000..f89cbb71e7 --- /dev/null +++ b/test/xds/main.go @@ -0,0 +1,60 @@ +package main + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "fmt" + "google.golang.org/grpc/credentials/insecure" + + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "google.golang.org/grpc" +) + +import ( + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" +) + + +const( + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" +) + +func main(){ + v3NodeProto := &v3corepb.Node{ + Id: "sidecar~10.195.0.134~random~local", + //Metadata: metadata, + //BuildVersion: gRPCVersion, + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + + nonNilCredsConfigV2 := &bootstrap.Config{ + XDSServer: &bootstrap.ServerConfig{ + ServerURI: "localhost:15010", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + //CredsType: "google_default", + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + + + xdsClient, err := client.NewWithConfig(nonNilCredsConfigV2) + if err != nil{ + panic(err) + } + + serviceName := "myService" + xdsClient.WatchListener(serviceName, func(update resource.ListenerUpdate, err error) { + fmt.Printf("%+v\n err = %s", update, err) + }) + select { + + } +} diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 2b9c734ee7..1b56d17996 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -22,12 +22,12 @@ package bootstrap import ( "bytes" + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" "encoding/json" "fmt" "io/ioutil" "strings" - internal "dubbo.apache.org/dubbo-go/v3/xds" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" @@ -333,7 +333,7 @@ func NewConfigFromContents(data []byte) (*Config, error) { return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) } configs := make(map[string]*certprovider.BuildableConfig) - getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + getBuilder := internal2.GetCertificateProviderBuilder.(func(string) certprovider.Builder) for instance, data := range providerInstances { var nameAndConfig struct { PluginName string `json:"plugin_name"` diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 37e4b99b44..05b982703f 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -19,13 +19,13 @@ package bootstrap import ( + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" "encoding/json" "errors" "fmt" "os" "testing" - internal "dubbo.apache.org/dubbo-go/v3/xds" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "github.com/golang/protobuf/proto" @@ -647,7 +647,7 @@ func TestNewConfigWithCertificateProviders(t *testing.T) { }`, } - getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + getBuilder := internal2.GetCertificateProviderBuilder.(func(string) certprovider.Builder) parser := getBuilder(fakeCertProviderName) if parser == nil { t.Fatalf("missing certprovider plugin %q", fakeCertProviderName) diff --git a/xds/client/client.go b/xds/client/client.go index 1d9192378e..50d1781c4a 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -27,9 +27,9 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" ) // clientImpl is the real implementation of the xds client. The exported Client diff --git a/xds/internal.go b/xds/internal/internal.go similarity index 99% rename from xds/internal.go rename to xds/internal/internal.go index 776b948bb6..1b596bf357 100644 --- a/xds/internal.go +++ b/xds/internal/internal.go @@ -18,7 +18,7 @@ // Package internal contains gRPC-internal code, to avoid polluting // the godoc of the top-level grpc package. It must not import any grpc // symbols to avoid circular dependencies. -package xds +package internal import ( "context" diff --git a/xds/server.go b/xds/server.go index caae913088..331d460991 100644 --- a/xds/server.go +++ b/xds/server.go @@ -20,6 +20,7 @@ package xds import ( "context" + "dubbo.apache.org/dubbo-go/v3/xds/internal" "errors" "fmt" "net" @@ -55,8 +56,8 @@ var ( return grpc.NewServer(opts...) } - grpcGetServerCreds = GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) - drainServerTransports = DrainServerTransports.(func(*grpc.Server, string)) + grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) + drainServerTransports = internal.DrainServerTransports.(func(*grpc.Server, string)) logger = grpclog.Component("xds") ) diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index 477ac7a0f4..aa1ef338d1 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -23,7 +23,7 @@ import ( "sync" "time" - "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index 30a3b4258f..a2a3b56ec3 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -20,11 +20,12 @@ package grpclog import ( + "dubbo.apache.org/dubbo-go/v3/common/logger" "os" ) // Logger is the logger used for the non-depth log functions. -var Logger LoggerV2 +var Logger = logger.GetLogger() // DepthLogger is the logger used for the depth log functions. var DepthLogger DepthLoggerV2 @@ -34,7 +35,7 @@ func InfoDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.InfoDepth(depth, args...) } else { - Logger.Infoln(args...) + Logger.Info(args...) } } @@ -43,7 +44,7 @@ func WarningDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.WarningDepth(depth, args...) } else { - Logger.Warningln(args...) + Logger.Warn(args...) } } @@ -52,7 +53,7 @@ func ErrorDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.ErrorDepth(depth, args...) } else { - Logger.Errorln(args...) + Logger.Error(args...) } } @@ -61,7 +62,7 @@ func FatalDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.FatalDepth(depth, args...) } else { - Logger.Fatalln(args...) + Logger.Fatal(args...) } os.Exit(1) } diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go index 82af70e96f..0083dbf0e7 100644 --- a/xds/utils/grpclog/prefixLogger.go +++ b/xds/utils/grpclog/prefixLogger.go @@ -63,9 +63,6 @@ func (pl *PrefixLogger) Errorf(format string, args ...interface{}) { // Debugf does info logging at verbose level 2. func (pl *PrefixLogger) Debugf(format string, args ...interface{}) { - if !Logger.V(2) { - return - } if pl != nil { // Handle nil, so the tests can pass in a nil logger. format = pl.prefix + format diff --git a/xds/utils/xds_cache/timeoutCache.go b/xds/utils/xds_cache/timeoutCache.go new file mode 100644 index 0000000000..d1c0c293a0 --- /dev/null +++ b/xds/utils/xds_cache/timeoutCache.go @@ -0,0 +1,143 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds_cache + +import ( + "sync" + "time" +) + +type cacheEntry struct { + item interface{} + // Note that to avoid deadlocks (potentially caused by lock ordering), + // callback can only be called without holding cache's mutex. + callback func() + timer *time.Timer + // deleted is set to true in Remove() when the call to timer.Stop() fails. + // This can happen when the timer in the cache entry fires around the same + // time that timer.stop() is called in Remove(). + deleted bool +} + +// TimeoutCache is a cache with items to be deleted after a timeout. +type TimeoutCache struct { + mu sync.Mutex + timeout time.Duration + cache map[interface{}]*cacheEntry +} + +// NewTimeoutCache creates a TimeoutCache with the given timeout. +func NewTimeoutCache(timeout time.Duration) *TimeoutCache { + return &TimeoutCache{ + timeout: timeout, + cache: make(map[interface{}]*cacheEntry), + } +} + +// Add adds an item to the cache, with the specified callback to be called when +// the item is removed from the cache upon timeout. If the item is removed from +// the cache using a call to Remove before the timeout expires, the callback +// will not be called. +// +// If the Add was successful, it returns (newly added item, true). If there is +// an existing entry for the specified key, the cache entry is not be updated +// with the specified item and it returns (existing item, false). +func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.cache[key]; ok { + return e.item, false + } + + entry := &cacheEntry{ + item: item, + callback: callback, + } + entry.timer = time.AfterFunc(c.timeout, func() { + c.mu.Lock() + if entry.deleted { + c.mu.Unlock() + // Abort the delete since this has been taken care of in Remove(). + return + } + delete(c.cache, key) + c.mu.Unlock() + entry.callback() + }) + c.cache[key] = entry + return item, true +} + +// Remove the item with the key from the cache. +// +// If the specified key exists in the cache, it returns (item associated with +// key, true) and the callback associated with the item is guaranteed to be not +// called. If the given key is not found in the cache, it returns (nil, false) +func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.removeInternal(key) + if !ok { + return nil, false + } + return entry.item, true +} + +// removeInternal removes and returns the item with key. +// +// caller must hold c.mu. +func (c *TimeoutCache) removeInternal(key interface{}) (*cacheEntry, bool) { + entry, ok := c.cache[key] + if !ok { + return nil, false + } + delete(c.cache, key) + if !entry.timer.Stop() { + // If stop was not successful, the timer has fired (this can only happen + // in a race). But the deleting function is blocked on c.mu because the + // mutex was held by the caller of this function. + // + // Set deleted to true to abort the deleting function. When the lock is + // released, the delete function will acquire the lock, check the value + // of deleted and return. + entry.deleted = true + } + return entry, true +} + +// Clear removes all entries, and runs the callbacks if runCallback is true. +func (c *TimeoutCache) Clear(runCallback bool) { + var entries []*cacheEntry + c.mu.Lock() + for key := range c.cache { + if e, ok := c.removeInternal(key); ok { + entries = append(entries, e) + } + } + c.mu.Unlock() + + if !runCallback { + return + } + + // removeInternal removes entries from cache, and also stops the timer, so + // the callback is guaranteed to be not called. If runCallback is true, + // manual execute all callbacks. + for _, entry := range entries { + entry.callback() + } +} From 24ae45363057dac0c84ce67edb388c0f49083916 Mon Sep 17 00:00:00 2001 From: "lizhixin.lzx" Date: Thu, 17 Mar 2022 23:50:09 +0800 Subject: [PATCH 06/19] Fix: init xds registry --- common/constant/key.go | 6 +- common/logger/logger.go | 13 +- registry/xds/registry.go | 155 +++++++++++++++++++++++ remoting/xds/client.go | 133 +++++++++++++++++++ test/xds/main.go | 48 +++++-- xds/client/resource/unmarshal_lds.go | 10 +- xds/utils/balancergroup/balancergroup.go | 2 +- xds/utils/grpclog/grpclog.go | 7 +- 8 files changed, 351 insertions(+), 23 deletions(-) create mode 100644 registry/xds/registry.go create mode 100644 remoting/xds/client.go diff --git a/common/constant/key.go b/common/constant/key.go index 0ab70b38c1..551cd151a0 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -228,7 +228,7 @@ const ( ) const ( - NacosKey = "nacos" + NacosKey = "nacos NacosGroupKey = "nacos.group" NacosDefaultRoleType = 3 NacosCacheDirKey = "nacos.cacheDir" @@ -275,6 +275,10 @@ const ( ZookeeperKey = "zookeeper" ) +const( + XDSRegistryKey = "xds" +) + const ( EtcdV3Key = "etcdv3" ) diff --git a/common/logger/logger.go b/common/logger/logger.go index 0af1ea861e..167dbf0fa9 100644 --- a/common/logger/logger.go +++ b/common/logger/logger.go @@ -42,6 +42,7 @@ type DubboLogger struct { type Config struct { LumberjackConfig *lumberjack.Logger `yaml:"lumberjack-config"` ZapConfig *zap.Config `yaml:"zap-config"` + CallerSkip int } // Logger is the interface for Logger types @@ -90,8 +91,16 @@ func InitLogger(conf *Config) { config.ZapConfig = conf.ZapConfig } + if conf != nil { + config.CallerSkip = conf.CallerSkip + } + + if config.CallerSkip == 0 { + config.CallerSkip = 1 + } + if conf == nil || conf.LumberjackConfig == nil { - zapLogger, _ = config.ZapConfig.Build(zap.AddCaller(), zap.AddCallerSkip(1)) + zapLogger, _ = config.ZapConfig.Build(zap.AddCaller(), zap.AddCallerSkip(config.CallerSkip)) } else { config.LumberjackConfig = conf.LumberjackConfig zapLogger = initZapLoggerWithSyncer(config) @@ -145,7 +154,7 @@ func initZapLoggerWithSyncer(conf *Config) *zap.Logger { zap.NewAtomicLevelAt(conf.ZapConfig.Level.Level()), ) - return zap.New(core, zap.AddCaller(), zap.AddCallerSkip(1)) + return zap.New(core, zap.AddCaller(), zap.AddCallerSkip(conf.CallerSkip)) } // getEncoder get encoder by config, zapcore support json and console encoder diff --git a/registry/xds/registry.go b/registry/xds/registry.go new file mode 100644 index 0000000000..1b775da631 --- /dev/null +++ b/registry/xds/registry.go @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + "bytes" + "dubbo.apache.org/dubbo-go/v3/remoting/xds" + "strconv" + "strings" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/extension" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/registry" +) + +var localIP = "" + +const ( + // RegistryConnDelay registry connection delay + RegistryConnDelay = 3 +) + +func init() { + localIP = common.GetLocalIp() + extension.SetRegistry(constant.XDSRegistryKey, newXDSRegistry) +} + +type xdsRegistry struct { + xdsWrappedClient *xds.WrappedClient + registryURL *common.URL +} + +func getCategory(url *common.URL) string { + role, _ := strconv.Atoi(url.GetParam(constant.RegistryRoleKey, strconv.Itoa(constant.NacosDefaultRoleType))) + category := common.DubboNodes[role] + return category +} + +func getServiceName(url *common.URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(getCategory(url))) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + +func appendParam(target *bytes.Buffer, url *common.URL, key string) { + value := url.GetParam(key, "") + target.Write([]byte(constant.NacosServiceNameSeparator)) + if strings.TrimSpace(value) != "" { + target.Write([]byte(value)) + } +} + +// Register will register the service @url to its nacos registry center +func (nr *xdsRegistry) Register(url *common.URL) error { + // todo get hostName of this app + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), "host name of this") +} + +// UnRegister +func (nr *xdsRegistry) UnRegister(url *common.URL) error { + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), "") +} + +// Subscribe from xds client +func (nr *xdsRegistry) Subscribe(url *common.URL, notifyListener registry.NotifyListener) error { + hostName, err := nr.getHostNameFromURL(url) + if err != nil { + return err + } + return nr.xdsWrappedClient.Subscribe(hostName, notifyListener) +} + +// UnSubscribe from xds client +func (nr *xdsRegistry) UnSubscribe(url *common.URL, _ registry.NotifyListener) error { + hostName, err := nr.getHostNameFromURL(url) + if err != nil { + return err + } + nr.xdsWrappedClient.UnSubscribe(hostName) + return nil +} + +// GetURL gets its registration URL +func (nr *xdsRegistry) GetURL() *common.URL { + return nr.registryURL +} + +func (nr *xdsRegistry) getHostNameFromURL(url *common.URL) (string, error) { + interfaceAppMap := nr.xdsWrappedClient.GetInterfaceAppNameMapFromPilot() + svcName := getServiceName(url) + hostName, ok := interfaceAppMap[svcName] + if !ok { + return "", perrors.Errorf("service %s not found", svcName) + } + return hostName, nil +} + +// IsAvailable determines nacos registry center whether it is available +func (nr *xdsRegistry) IsAvailable() bool { + // TODO + return true +} + +// nolint +func (nr *xdsRegistry) Destroy() { + // todo unregistry all + //for _, url := range nr.registryUrls { + // err := nr.UnRegister(url) + // logger.Infof("DeRegister Nacos URL:%+v", url) + // if err != nil { + // logger.Errorf("Deregister URL:%+v err:%v", url, err.Error()) + // } + //} + return +} + +// newXDSRegistry will create new instance +func newXDSRegistry(url *common.URL) (registry.Registry, error) { + logger.Infof("[XDS Registry] New nacos registry with url = %+v", url.ToMap()) + // todo get podName, get Namespace Name + + wrappedXDSClient := xds.NewXDSWrappedClient("", "", localIP, url.Location) + tmpRegistry := &xdsRegistry{ + xdsWrappedClient: wrappedXDSClient, + registryURL: url, + } + return tmpRegistry, nil +} diff --git a/remoting/xds/client.go b/remoting/xds/client.go new file mode 100644 index 0000000000..2bb08d50a7 --- /dev/null +++ b/remoting/xds/client.go @@ -0,0 +1,133 @@ +package xds + +import ( + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "encoding/json" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + structpb "github.com/golang/protobuf/ptypes/struct" + perrors "github.com/pkg/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "sync" +) + +const ( + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" +) + +type WrappedClient struct { + interfaceAppNameMap map[string]string + subscribeStopChMap sync.Map + podName string + namespace string + localIP string + istiodAddr string + xdsClient client.XDSClient + lock sync.Mutex +} + +func NewXDSWrappedClient(podName, namspace, localIP, istiodAddr string) *WrappedClient { + return &WrappedClient{ + podName: podName, + namespace: namspace, + localIP: localIP, + istiodAddr: istiodAddr, + } +} + +func (w *WrappedClient) GetInterfaceAppNameMapFromPilot() map[string]string { + // todo get map from + // http://istiodAddr:8080/debug/adsz + return nil +} + +func (w *WrappedClient) Subscribe(hostName string, lst registry.NotifyListener) error { + _, ok := w.subscribeStopChMap.Load(hostName) + if ok { + return perrors.Errorf("XDS WrappedClient subscribe hostName failed, subscription already exist.") + } + stopCh := make(chan struct{}) + w.subscribeStopChMap.Store(hostName, stopCh) +LOOP: + for { + // todo cds, eds + select { + case <-stopCh: + break LOOP + //case <- eventCh: + default: + } + + // todo to invoker + + // todo notify + lst.Notify(®istry.ServiceEvent{}) + } + return nil +} + +func (w *WrappedClient) UnSubscribe(hostName string) { + if stopCh, ok := w.subscribeStopChMap.Load(hostName); ok { + close(stopCh.(chan struct{})) + } +} + +func (w *WrappedClient) interfaceAppNameMap2String() string { + data, _ := json.Marshal(w.interfaceAppNameMap) + return string(data) +} + +// ChangeInterfaceMap change the map of interfaceName -> appname, if appName is empty, delete this interfaceName +func (w *WrappedClient) ChangeInterfaceMap(interfaceName, appName string) error { + w.lock.Lock() + defer w.lock.Unlock() + if w.xdsClient != nil { + w.xdsClient.Close() + } + if appName == "" { + delete(w.interfaceAppNameMap, interfaceName) + } else { + w.interfaceAppNameMap[interfaceName] = appName + } + + v3NodeProto := &v3corepb.Node{ + Id: "sidecar~" + w.localIP + "~" + w.podName + "." + w.namespace + "~" + w.namespace + ".svc.cluster.local", + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "LABELS": { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "DUBBO_GO": { + Kind: &structpb.Value_StringValue{StringValue: w.interfaceAppNameMap2String()}, + }, + }, + }}, + }, + }, + }, + } + + nonNilCredsConfigV2 := &bootstrap.Config{ + XDSServer: &bootstrap.ServerConfig{ + ServerURI: w.istiodAddr + "15010", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + + xdsClient, err := client.NewWithConfig(nonNilCredsConfigV2) + if err != nil { + return err + } + w.xdsClient = xdsClient + return nil +} diff --git a/test/xds/main.go b/test/xds/main.go index f89cbb71e7..6375b7d3d4 100644 --- a/test/xds/main.go +++ b/test/xds/main.go @@ -6,6 +6,7 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "fmt" + structpb "github.com/golang/protobuf/ptypes/struct" "google.golang.org/grpc/credentials/insecure" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -17,26 +18,41 @@ import ( _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" ) - -const( +const ( gRPCUserAgentName = "gRPC Go" clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) -func main(){ +// ATTENTION! export GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT=false +func main() { v3NodeProto := &v3corepb.Node{ - Id: "sidecar~10.195.0.134~random~local", - //Metadata: metadata, - //BuildVersion: gRPCVersion, + Id: "sidecar~172.1.1.1~sleep-55b5877479-rwcct.default~default.svc.cluster.local", UserAgentName: gRPCUserAgentName, + Cluster: "testCluster", UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, ClientFeatures: []string{clientFeatureNoOverprovisioning}, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "LABELS": { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "label1": { + Kind: &structpb.Value_StringValue{StringValue: "val1"}, + }, + "label2": { + Kind: &structpb.Value_StringValue{StringValue: "val2"}, + }, + }, + }}, + }, + }, + }, } nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ ServerURI: "localhost:15010", - Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), //CredsType: "google_default", TransportAPI: version.TransportV3, NodeProto: v3NodeProto, @@ -44,17 +60,23 @@ func main(){ ClientDefaultListenerResourceNameTemplate: "%s", } - xdsClient, err := client.NewWithConfig(nonNilCredsConfigV2) - if err != nil{ + if err != nil { panic(err) } - serviceName := "myService" - xdsClient.WatchListener(serviceName, func(update resource.ListenerUpdate, err error) { + clusterName := "outbound|20000||dubbo-go-app.default.svc.cluster.local" // + //clusterName := "outbound|27||laurence-svc.default.svc.cluster.local" + xdsClient.WatchListener("", func(update resource.ListenerUpdate, err error) { fmt.Printf("%+v\n err = %s", update, err) }) - select { - } + xdsClient.WatchCluster(clusterName, func(update resource.ClusterUpdate, err error) { + fmt.Printf("%+v\n err = %s", update, err) + }) + + xdsClient.WatchEndpoints(clusterName, func(update resource.EndpointsUpdate, err error) { + fmt.Printf("%+v\n err = %s", update, err) + }) + select {} } diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index 4e949e0f05..5e2e246b52 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -288,10 +288,10 @@ func processServerSideListener(lis *v3listenerpb.Listener, logger *grpclog.Prefi }, } - fcMgr, err := NewFilterChainManager(lis, logger) - if err != nil { - return nil, err - } - lu.InboundListenerCfg.FilterChains = fcMgr + //fcMgr, err := NewFilterChainManager(lis, logger) + //if err != nil { + // return nil, err + //} + //lu.InboundListenerCfg.FilterChains = fcMgr return lu, nil } diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index aa1ef338d1..fed83e3159 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -23,8 +23,8 @@ import ( "sync" "time" - cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" "google.golang.org/grpc/balancer" "google.golang.org/grpc/connectivity" "google.golang.org/grpc/resolver" diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index a2a3b56ec3..ba5874c21e 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -25,7 +25,12 @@ import ( ) // Logger is the logger used for the non-depth log functions. -var Logger = logger.GetLogger() +var Logger = func() logger.Logger { + logger.InitLogger(&logger.Config{ + CallerSkip: 2, + }) + return logger.GetLogger() +}() // DepthLogger is the logger used for the depth log functions. var DepthLogger DepthLoggerV2 From c8aed729721bfe411d63fe584202c49692342ccd Mon Sep 17 00:00:00 2001 From: "lizhixin.lzx" Date: Sun, 20 Mar 2022 23:11:08 +0800 Subject: [PATCH 07/19] Fix: init xds registry --- common/constant/env.go | 3 + common/constant/key.go | 4 +- go.sum | 2 - imports/imports.go | 3 + protocol/dubbo3/dubbo3_invoker.go | 2 +- registry/xds/registry.go | 61 ++-- remoting/xds/client.go | 283 ++++++++++++++---- remoting/xds/debug.go | 25 ++ test/xds/main.go | 41 ++- xds/balancer/balancer.go | 5 +- xds/balancer/cdsbalancer/cdsbalancer.go | 23 +- xds/balancer/cdsbalancer/cluster_handler.go | 2 + xds/balancer/cdsbalancer/logging.go | 7 +- xds/balancer/clusterimpl/clusterimpl.go | 18 +- xds/balancer/clusterimpl/config.go | 7 +- xds/balancer/clusterimpl/config_test.go | 8 +- xds/balancer/clusterimpl/logging.go | 7 +- xds/balancer/clusterimpl/picker.go | 13 +- .../clustermanager/balancerstateaggregator.go | 8 +- xds/balancer/clustermanager/clustermanager.go | 16 +- xds/balancer/clustermanager/config.go | 7 +- xds/balancer/clustermanager/picker.go | 4 + .../clusterresolver/clusterresolver.go | 21 +- xds/balancer/clusterresolver/config.go | 10 +- xds/balancer/clusterresolver/configbuilder.go | 14 +- xds/balancer/clusterresolver/logging.go | 7 +- .../clusterresolver/resource_resolver.go | 2 + .../clusterresolver/resource_resolver_dns.go | 3 + .../clusterresolver/weightedtarget_config.go | 8 +- xds/balancer/loadstore/load_store_wrapper.go | 2 + xds/balancer/orca/orca.go | 8 +- xds/balancer/priority/balancer.go | 13 +- xds/balancer/priority/balancer_child.go | 3 + xds/balancer/priority/balancer_priority.go | 3 + xds/balancer/priority/config.go | 7 +- xds/balancer/priority/config_test.go | 8 +- xds/balancer/priority/ignore_resolve_now.go | 3 + xds/balancer/priority/logging.go | 7 +- xds/balancer/priority/utils_test.go | 4 +- xds/balancer/ringhash/config.go | 2 + xds/balancer/ringhash/config_test.go | 2 + xds/balancer/ringhash/logging.go | 7 +- xds/balancer/ringhash/picker.go | 10 +- xds/balancer/ringhash/ring.go | 3 + xds/balancer/ringhash/ring_test.go | 3 + xds/balancer/ringhash/ringhash.go | 12 +- xds/balancer/ringhash/util.go | 4 +- xds/client/attributes.go | 9 +- xds/client/authority.go | 10 + xds/client/bootstrap/bootstrap.go | 18 +- xds/client/bootstrap/bootstrap_test.go | 19 +- xds/client/bootstrap/logging.go | 5 +- xds/client/bootstrap/template_test.go | 4 +- xds/client/client.go | 23 +- xds/client/controller.go | 5 + xds/client/controller/controller.go | 50 +++- xds/client/controller/loadreport.go | 7 +- xds/client/controller/transport.go | 10 +- xds/client/controller/version/v2/client.go | 34 ++- .../controller/version/v2/loadreport.go | 18 +- xds/client/controller/version/v3/client.go | 32 +- .../controller/version/v3/loadreport.go | 18 +- xds/client/controller/version/version.go | 25 +- xds/client/load/store_test.go | 2 + xds/client/logging.go | 7 +- xds/client/pubsub/dump.go | 5 +- xds/client/pubsub/interface.go | 4 +- xds/client/pubsub/pubsub.go | 2 + xds/client/pubsub/update.go | 14 +- xds/client/pubsub/watch.go | 6 + xds/client/resource/errors.go | 4 +- xds/client/resource/filter_chain.go | 16 +- xds/client/resource/locality_id.go | 2 + xds/client/resource/matcher.go | 7 +- xds/client/resource/matcher_path.go | 2 + xds/client/resource/name.go | 2 + xds/client/resource/type.go | 7 +- xds/client/resource/type_cds.go | 4 +- xds/client/resource/type_lds.go | 7 +- xds/client/resource/type_rds.go | 10 +- xds/client/resource/unmarshal.go | 7 +- xds/client/resource/unmarshal_cds.go | 23 +- xds/client/resource/unmarshal_eds.go | 11 +- xds/client/resource/unmarshal_lds.go | 17 +- xds/client/resource/unmarshal_rds.go | 18 +- xds/client/singleton.go | 2 + xds/csds/csds.go | 13 +- xds/httpfilter/fault/fault.go | 21 +- xds/httpfilter/httpfilter.go | 5 +- xds/httpfilter/rbac/rbac.go | 18 +- xds/httpfilter/router/router.go | 12 +- xds/internal/internal.go | 3 + xds/resolver/logging.go | 7 +- xds/resolver/serviceconfig.go | 16 +- xds/resolver/watch_service.go | 2 + xds/resolver/xds_resolver.go | 10 +- xds/server.go | 21 +- xds/server/conn_wrapper.go | 7 +- xds/server/listener_wrapper.go | 13 +- xds/server/rds_handler.go | 2 + xds/server_options.go | 2 + xds/utils/backoff/backoff.go | 7 +- xds/utils/balancer/stub/stub.go | 4 +- xds/utils/balancergroup/balancergroup.go | 11 +- xds/utils/buffer/unbounded.go | 4 +- xds/utils/credentials/xds/handshake_info.go | 9 +- .../credentials/xds/handshake_info_test.go | 2 + xds/utils/grpclog/grpclog.go | 5 +- xds/utils/grpcutil/metadata.go | 2 + xds/utils/grpcutil/regex.go | 4 +- xds/utils/hierarchy/hierarchy_test.go | 4 + xds/utils/matcher/matcher_header.go | 5 +- xds/utils/matcher/matcher_header_test.go | 2 + xds/utils/matcher/regex.go | 4 +- xds/utils/matcher/string_matcher.go | 7 +- xds/utils/matcher/string_matcher_test.go | 3 + xds/utils/metadata/metadata.go | 2 + xds/utils/pretty/pretty.go | 4 + xds/utils/rbac/matchers.go | 7 +- xds/utils/rbac/rbac_engine.go | 8 +- xds/utils/resolver/config_selector.go | 8 +- xds/utils/resolver/passthrough/passthrough.go | 4 +- xds/utils/resolver/unix/unix.go | 7 +- xds/utils/serviceconfig/serviceconfig.go | 5 + xds/utils/serviceconfig/serviceconfig_test.go | 4 + xds/utils/wrr/random.go | 2 + xds/xds_handshake_cluster.go | 1 + 127 files changed, 1157 insertions(+), 301 deletions(-) create mode 100644 remoting/xds/debug.go diff --git a/common/constant/env.go b/common/constant/env.go index 9e20a6d520..511b7325c9 100644 --- a/common/constant/env.go +++ b/common/constant/env.go @@ -23,4 +23,7 @@ const ( ConfigFileEnvKey = "DUBBO_GO_CONFIG_PATH" // AppLogConfFile ... AppLogConfFile = "AppLogConfFile" + + PodNameEnvKey = "POD_NAME" + PodNamespaceEnvKey = "POD_NAMESPACE" ) diff --git a/common/constant/key.go b/common/constant/key.go index 551cd151a0..0e1a684d3f 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -228,7 +228,7 @@ const ( ) const ( - NacosKey = "nacos + NacosKey = "nacos" NacosGroupKey = "nacos.group" NacosDefaultRoleType = 3 NacosCacheDirKey = "nacos.cacheDir" @@ -275,7 +275,7 @@ const ( ZookeeperKey = "zookeeper" ) -const( +const ( XDSRegistryKey = "xds" ) diff --git a/go.sum b/go.sum index 481591584b..dc1bbb3fee 100644 --- a/go.sum +++ b/go.sum @@ -1306,8 +1306,6 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/imports/imports.go b/imports/imports.go index 3251ca61df..ab40b61968 100644 --- a/imports/imports.go +++ b/imports/imports.go @@ -72,5 +72,8 @@ import ( _ "dubbo.apache.org/dubbo-go/v3/registry/polaris" _ "dubbo.apache.org/dubbo-go/v3/registry/protocol" _ "dubbo.apache.org/dubbo-go/v3/registry/servicediscovery" + _ "dubbo.apache.org/dubbo-go/v3/registry/xds" _ "dubbo.apache.org/dubbo-go/v3/registry/zookeeper" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" ) diff --git a/protocol/dubbo3/dubbo3_invoker.go b/protocol/dubbo3/dubbo3_invoker.go index 1e33bf2eb0..b87c0de97c 100644 --- a/protocol/dubbo3/dubbo3_invoker.go +++ b/protocol/dubbo3/dubbo3_invoker.go @@ -230,9 +230,9 @@ func (di *DubboInvoker) getTimeout(invocation *invocation_impl.RPCInvocation) ti func (di *DubboInvoker) IsAvailable() bool { client := di.getClient() if client != nil { + // FIXME here can't check if tcp server is started now!!! return client.IsAvailable() } - return false } diff --git a/registry/xds/registry.go b/registry/xds/registry.go index 1b775da631..b0050da665 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -19,7 +19,7 @@ package xds import ( "bytes" - "dubbo.apache.org/dubbo-go/v3/remoting/xds" + "os" "strconv" "strings" ) @@ -34,6 +34,7 @@ import ( "dubbo.apache.org/dubbo-go/v3/common/extension" "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/remoting/xds" ) var localIP = "" @@ -53,6 +54,10 @@ type xdsRegistry struct { registryURL *common.URL } +func isProvider(url *common.URL) bool { + return getCategory(url) == "providers" +} + func getCategory(url *common.URL) string { role, _ := strconv.Atoi(url.GetParam(constant.RegistryRoleKey, strconv.Itoa(constant.NacosDefaultRoleType))) category := common.DubboNodes[role] @@ -69,6 +74,16 @@ func getServiceName(url *common.URL) string { return buffer.String() } +func getSubscribeName(url *common.URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(common.DubboNodes[common.PROVIDER])) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + func appendParam(target *bytes.Buffer, url *common.URL, key string) { value := url.GetParam(key, "") target.Write([]byte(constant.NacosServiceNameSeparator)) @@ -79,31 +94,36 @@ func appendParam(target *bytes.Buffer, url *common.URL, key string) { // Register will register the service @url to its nacos registry center func (nr *xdsRegistry) Register(url *common.URL) error { - // todo get hostName of this app - return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), "host name of this") + if !isProvider(url) { + return nil + } + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), true) } // UnRegister func (nr *xdsRegistry) UnRegister(url *common.URL) error { - return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), "") + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), false) } // Subscribe from xds client func (nr *xdsRegistry) Subscribe(url *common.URL, notifyListener registry.NotifyListener) error { - hostName, err := nr.getHostNameFromURL(url) + if isProvider(url) { + return nil + } + hostAddr, svcAddr, err := nr.getHostAddrFromURL(url) if err != nil { return err } - return nr.xdsWrappedClient.Subscribe(hostName, notifyListener) + return nr.xdsWrappedClient.Subscribe(svcAddr, url.GetParam(constant.InterfaceKey, ""), hostAddr, notifyListener) } // UnSubscribe from xds client func (nr *xdsRegistry) UnSubscribe(url *common.URL, _ registry.NotifyListener) error { - hostName, err := nr.getHostNameFromURL(url) + _, svcAddr, err := nr.getHostAddrFromURL(url) if err != nil { return err } - nr.xdsWrappedClient.UnSubscribe(hostName) + nr.xdsWrappedClient.UnSubscribe(svcAddr) return nil } @@ -112,14 +132,10 @@ func (nr *xdsRegistry) GetURL() *common.URL { return nr.registryURL } -func (nr *xdsRegistry) getHostNameFromURL(url *common.URL) (string, error) { - interfaceAppMap := nr.xdsWrappedClient.GetInterfaceAppNameMapFromPilot() - svcName := getServiceName(url) - hostName, ok := interfaceAppMap[svcName] - if !ok { - return "", perrors.Errorf("service %s not found", svcName) - } - return hostName, nil +func (nr *xdsRegistry) getHostAddrFromURL(url *common.URL) (string, string, error) { + svcName := getSubscribeName(url) + hostAddr, err := nr.xdsWrappedClient.GetHostAddrFromPilot(svcName) + return hostAddr, svcName, err } // IsAvailable determines nacos registry center whether it is available @@ -143,10 +159,17 @@ func (nr *xdsRegistry) Destroy() { // newXDSRegistry will create new instance func newXDSRegistry(url *common.URL) (registry.Registry, error) { - logger.Infof("[XDS Registry] New nacos registry with url = %+v", url.ToMap()) - // todo get podName, get Namespace Name + logger.Infof("[XDS Registry] New XDS registry with url = %+v", url.ToMap()) + pn := os.Getenv(constant.PodNameEnvKey) + ns := os.Getenv(constant.PodNamespaceEnvKey) + if pn == "" || ns == "" { + return nil, perrors.New("POD_NAME and POD_NAMESPACE can't be empty when using xds registry") + } - wrappedXDSClient := xds.NewXDSWrappedClient("", "", localIP, url.Location) + wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, url.Ip) + if err != nil { + return nil, err + } tmpRegistry := &xdsRegistry{ xdsWrappedClient: wrappedXDSClient, registryURL: url, diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 2bb08d50a7..c80289c603 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -1,17 +1,36 @@ package xds import ( - "dubbo.apache.org/dubbo-go/v3/registry" - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "sync" + "time" +) + +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + structpb "github.com/golang/protobuf/ptypes/struct" + perrors "github.com/pkg/errors" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "sync" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/remoting" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" ) const ( @@ -24,56 +43,111 @@ type WrappedClient struct { subscribeStopChMap sync.Map podName string namespace string - localIP string - istiodAddr string + localIP string // to find hostAddr by cds and eds + istiodHostName string // todo: here must be istiod-grpc.istio-system.svc.cluster.local + hostAddr string // dubbo-go-app.default.svc.cluster.local:20000 + istiodPodIP string // to call istiod unexposed debug port 8080 xdsClient client.XDSClient lock sync.Mutex + + endpointClusterMap sync.Map } -func NewXDSWrappedClient(podName, namspace, localIP, istiodAddr string) *WrappedClient { - return &WrappedClient{ - podName: podName, - namespace: namspace, - localIP: localIP, - istiodAddr: istiodAddr, +func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*WrappedClient, error) { + // get hostname from http://localhost:8080/debug/endpointz + newClient := &WrappedClient{ + podName: podName, + namespace: namespace, + localIP: localIP, + istiodHostName: istiodHostName, + interfaceAppNameMap: make(map[string]string), + } + if err := newClient.initClientAndloadLocalHostAddr(); err != nil { + return nil, err } + return newClient, nil } -func (w *WrappedClient) GetInterfaceAppNameMapFromPilot() map[string]string { - // todo get map from - // http://istiodAddr:8080/debug/adsz - return nil -} +func (w *WrappedClient) getInterfaceHostAddrMapFromPilot() (map[string]string, error) { + req, _ := http.NewRequest(http.MethodGet, "http://"+w.istiodPodIP+":8080/debug/adsz", nil) + token, err := os.ReadFile("/var/run/secrets/token/istio-token") + if err != nil { + return nil, err + } + req.Header.Add("Authorization", "Bearer "+string(token)) + rsp, err := http.DefaultClient.Do(req) + if err != nil { + logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio %s with error %s\n", w.istiodHostName, err) + return nil, err + } -func (w *WrappedClient) Subscribe(hostName string, lst registry.NotifyListener) error { - _, ok := w.subscribeStopChMap.Load(hostName) - if ok { - return perrors.Errorf("XDS WrappedClient subscribe hostName failed, subscription already exist.") + data, err := ioutil.ReadAll(rsp.Body) + adszRsp := &ADSZResponse{} + if err := json.Unmarshal(data, adszRsp); err != nil { + return nil, err } - stopCh := make(chan struct{}) - w.subscribeStopChMap.Store(hostName, stopCh) -LOOP: + return adszRsp.GetMap(), nil +} + +// GetHostAddrFromPilot todo 1. timeout 2. hostAddr change? +func (w *WrappedClient) GetHostAddrFromPilot(serviceKey string) (string, error) { for { - // todo cds, eds - select { - case <-stopCh: - break LOOP - //case <- eventCh: - default: + if interfaceHostMap, err := w.getInterfaceHostAddrMapFromPilot(); err != nil { + return "", err + } else { + hostName, ok := interfaceHostMap[serviceKey] + if !ok { + logger.Infof("[XDS Wrapped Client] Try getting service %s 's host from istio %d\n", serviceKey, w.istiodHostName) + time.Sleep(time.Millisecond * 100) + continue + } + return hostName, nil } + } +} - // todo to invoker - - // todo notify - lst.Notify(®istry.ServiceEvent{}) +func (w *WrappedClient) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { + _, ok := w.subscribeStopChMap.Load(svcUniqueName) + if ok { + return perrors.Errorf("XDS WrappedClient subscribe interface %s failed, subscription already exist.", interfaceName) } + stopCh := make(chan struct{}) + w.subscribeStopChMap.Store(svcUniqueName, stopCh) + ipPort := strings.Split(hostAddr, ":") + hostName := ipPort[0] + port := ipPort[1] + // todo cds, eds + cancel := w.xdsClient.WatchEndpoints(fmt.Sprintf("outbound|%s||%s", port, hostName), func(update resource.EndpointsUpdate, err error) { + for _, v := range update.Localities { + for _, e := range v.Endpoints { + // todo 1. register protocol in server side metadata 2. get metadata from endpoint, for router + url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", e.Address, interfaceName)) + logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", + interfaceName, e.Address, e.HealthStatus) + if e.HealthStatus == resource.EndpointHealthStatusHealthy { + lst.Notify(®istry.ServiceEvent{ + Action: remoting.EventTypeUpdate, + Service: url, + }) + } else { + lst.Notify(®istry.ServiceEvent{ + Action: remoting.EventTypeDel, + Service: url, + }) + } + } + } + }) + <-stopCh + cancel() return nil } -func (w *WrappedClient) UnSubscribe(hostName string) { - if stopCh, ok := w.subscribeStopChMap.Load(hostName); ok { +func (w *WrappedClient) UnSubscribe(svcUniqueName string) { + if stopCh, ok := w.subscribeStopChMap.Load(svcUniqueName); ok { close(stopCh.(chan struct{})) } + w.subscribeStopChMap.Delete(svcUniqueName) } func (w *WrappedClient) interfaceAppNameMap2String() string { @@ -81,42 +155,105 @@ func (w *WrappedClient) interfaceAppNameMap2String() string { return string(data) } -// ChangeInterfaceMap change the map of interfaceName -> appname, if appName is empty, delete this interfaceName -func (w *WrappedClient) ChangeInterfaceMap(interfaceName, appName string) error { +// ChangeInterfaceMap change the map of interfaceName -> appname, if add is true, register, else unregister +func (w *WrappedClient) ChangeInterfaceMap(interfaceName string, add bool) error { w.lock.Lock() defer w.lock.Unlock() - if w.xdsClient != nil { - w.xdsClient.Close() - } - if appName == "" { - delete(w.interfaceAppNameMap, interfaceName) + if add { + w.interfaceAppNameMap[interfaceName] = w.hostAddr } else { - w.interfaceAppNameMap[interfaceName] = appName + delete(w.interfaceAppNameMap, interfaceName) + } + if w.xdsClient == nil { + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + if err != nil { + return err + } + w.xdsClient = xdsClient + return nil + } + + if err := w.xdsClient.SetMetadata(getDubboGoMetadata(w.interfaceAppNameMap2String())); err != nil { + return err } + return nil +} +func (w *WrappedClient) initClientAndloadLocalHostAddr() error { + // call watch and refresh istiod debug interface + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + if err != nil { + return err + } + stopCh := make(chan struct{}) + foundLocal := false + foundIstiod := false + cancel := xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + if update.ClusterName == "" { + return + } + clusterNameList := strings.Split(update.ClusterName, "|") + // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! + if clusterNameList[3] == w.istiodHostName { + // 1. find istiod podIP + // todo: When would eds level watch be cancelled? + _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + if foundIstiod { + return + } + for _, v := range endpoint.Localities { + for _, e := range v.Endpoints { + w.endpointClusterMap.Store(e.Address, update.ClusterName) + addrs := strings.Split(e.Address, ":") + w.istiodPodIP = addrs[0] + foundIstiod = true + if foundLocal && foundIstiod { + stopCh <- struct{}{} + } + } + } + }) + return + } + // 2. found local hostAddr + // todo: When would eds level watch be cancelled? + _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + if foundLocal { + return + } + for _, v := range endpoint.Localities { + for _, e := range v.Endpoints { + w.endpointClusterMap.Store(e.Address, update.ClusterName) + addrs := strings.Split(e.Address, ":") + if addrs[0] == w.localIP { + clusterNames := strings.Split(update.ClusterName, "|") + w.hostAddr = clusterNames[3] + ":" + clusterNames[1] + foundLocal = true + } + if foundLocal && foundIstiod { + stopCh <- struct{}{} + } + } + } + }) + }) + <-stopCh + cancel() + w.xdsClient = xdsClient + return nil +} + +func newxdsClient(localIP, podName, namespace, dubboGoMetadata, istiodIP string) (client.XDSClient, error) { v3NodeProto := &v3corepb.Node{ - Id: "sidecar~" + w.localIP + "~" + w.podName + "." + w.namespace + "~" + w.namespace + ".svc.cluster.local", + Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", UserAgentName: gRPCUserAgentName, UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, ClientFeatures: []string{clientFeatureNoOverprovisioning}, - Metadata: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "LABELS": { - Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - "DUBBO_GO": { - Kind: &structpb.Value_StringValue{StringValue: w.interfaceAppNameMap2String()}, - }, - }, - }}, - }, - }, - }, } nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ - ServerURI: w.istiodAddr + "15010", + ServerURI: istiodIP + ":15010", Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), TransportAPI: version.TransportV3, NodeProto: v3NodeProto, @@ -124,10 +261,28 @@ func (w *WrappedClient) ChangeInterfaceMap(interfaceName, appName string) error ClientDefaultListenerResourceNameTemplate: "%s", } - xdsClient, err := client.NewWithConfig(nonNilCredsConfigV2) + newClient, err := client.NewWithConfig(nonNilCredsConfigV2) if err != nil { - return err + return nil, err + } + if err := newClient.SetMetadata(getDubboGoMetadata(dubboGoMetadata)); err != nil { + return nil, err + } + return newClient, nil +} + +func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { + return &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "LABELS": { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "DUBBO_GO": { + Kind: &structpb.Value_StringValue{StringValue: dubboGoMetadata}, + }, + }, + }}, + }, + }, } - w.xdsClient = xdsClient - return nil } diff --git a/remoting/xds/debug.go b/remoting/xds/debug.go new file mode 100644 index 0000000000..02e23789ad --- /dev/null +++ b/remoting/xds/debug.go @@ -0,0 +1,25 @@ +package xds + +import ( + "encoding/json" +) + +type ADSZResponse struct { + Clients []ADSZClient `json:"clients"` +} + +type ADSZClient struct { + Metadata map[string]interface{} `json:"metadata"` +} + +func (a *ADSZResponse) GetMap() map[string]string { + result := make(map[string]string) + for _, c := range a.Clients { + resultMap := make(map[string]string) + json.Unmarshal([]byte(c.Metadata["LABELS"].(map[string]interface{})["DUBBO_GO"].(string)), &resultMap) + for k, v := range resultMap { + result[k] = v + } + } + return result +} diff --git a/test/xds/main.go b/test/xds/main.go index 6375b7d3d4..35d45cf1a1 100644 --- a/test/xds/main.go +++ b/test/xds/main.go @@ -1,21 +1,25 @@ package main import ( - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "fmt" - structpb "github.com/golang/protobuf/ptypes/struct" - "google.golang.org/grpc/credentials/insecure" +) +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + structpb "github.com/golang/protobuf/ptypes/struct" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" ) const ( @@ -65,18 +69,29 @@ func main() { panic(err) } - clusterName := "outbound|20000||dubbo-go-app.default.svc.cluster.local" // - //clusterName := "outbound|27||laurence-svc.default.svc.cluster.local" - xdsClient.WatchListener("", func(update resource.ListenerUpdate, err error) { - fmt.Printf("%+v\n err = %s", update, err) - }) + //clusterName := "outbound|20000||dubbo-go-app.default.svc.cluster.local" // + //clusterName := "outbound|8848||nacos.default.svc.cluster.local" + //endpointClusterMap := sync.Map{} + //xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + // xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + // for _, v := range endpoint.Localities { + // for _, e := range v.Endpoints { + // endpointClusterMap.Store(e.Address, update.ClusterName) + // } + // } + // }) + //}) - xdsClient.WatchCluster(clusterName, func(update resource.ClusterUpdate, err error) { + // + xdsClient.WatchEndpoints("outbound|15012||istiod.istio-system.svc.cluster.local", func(update resource.EndpointsUpdate, err error) { fmt.Printf("%+v\n err = %s", update, err) }) - xdsClient.WatchEndpoints(clusterName, func(update resource.EndpointsUpdate, err error) { + xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { fmt.Printf("%+v\n err = %s", update, err) + }) + // + select {} } diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go index 5ac911301d..490124cab5 100644 --- a/xds/balancer/balancer.go +++ b/xds/balancer/balancer.go @@ -19,11 +19,14 @@ // Package balancer installs all the xds balancers. package balancer +import ( + _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer +) + import ( _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/cdsbalancer" // Register the CDS balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" // Register the xds_cluster_impl balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" // Register the xds_cluster_manager balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" // Register the xds_cluster_resolver balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" // Register the priority balancer - _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer ) diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index 9ab069e1de..ab95d5687f 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -21,7 +21,23 @@ import ( "encoding/json" "errors" "fmt" +) + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client" @@ -32,13 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/tls/certprovider" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) const ( diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go index 137825309b..dec61af342 100644 --- a/xds/balancer/cdsbalancer/cluster_handler.go +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -19,7 +19,9 @@ package cdsbalancer import ( "errors" "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go index e0d21c0c97..e19b4d0cad 100644 --- a/xds/balancer/cdsbalancer/logging.go +++ b/xds/balancer/cdsbalancer/logging.go @@ -20,11 +20,16 @@ package cdsbalancer import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[cds-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go index 0be70ec12f..c79a866dd7 100644 --- a/xds/balancer/clusterimpl/clusterimpl.go +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -24,12 +24,24 @@ package clusterimpl import ( - internal "dubbo.apache.org/dubbo-go/v3/xds" "encoding/json" "fmt" "sync" "sync/atomic" +) + +import ( + "google.golang.org/grpc/balancer" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( + internal "dubbo.apache.org/dubbo-go/v3/xds" "dubbo.apache.org/dubbo-go/v3/xds/balancer/loadstore" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/load" @@ -38,10 +50,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) const ( diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go index a33ef0472a..9fe1a41a7e 100644 --- a/xds/balancer/clusterimpl/config.go +++ b/xds/balancer/clusterimpl/config.go @@ -20,11 +20,16 @@ package clusterimpl import ( "encoding/json" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // DropConfig contains the category, and drop ratio. type DropConfig struct { Category string diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go index b217a32119..0bdaa66cf9 100644 --- a/xds/balancer/clusterimpl/config_test.go +++ b/xds/balancer/clusterimpl/config_test.go @@ -20,14 +20,20 @@ package clusterimpl import ( "testing" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" _ "google.golang.org/grpc/balancer/roundrobin" _ "google.golang.org/grpc/balancer/weightedtarget" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + const ( testJSONConfig = `{ "cluster": "test_cluster", diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go index 70c8d2dad1..b94aa44256 100644 --- a/xds/balancer/clusterimpl/logging.go +++ b/xds/balancer/clusterimpl/logging.go @@ -20,11 +20,16 @@ package clusterimpl import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-cluster-impl-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go index 5e3da761fd..39874a6f86 100644 --- a/xds/balancer/clusterimpl/picker.go +++ b/xds/balancer/clusterimpl/picker.go @@ -19,16 +19,23 @@ package clusterimpl import ( - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" +) + // NewRandomWRR is used when calculating drops. It's exported so that tests can // override it. var NewRandomWRR = wrr.NewRandom diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go index aec4fb658c..8b0ad174df 100644 --- a/xds/balancer/clustermanager/balancerstateaggregator.go +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -21,13 +21,19 @@ package clustermanager import ( "fmt" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + type subBalancerState struct { state balancer.State // stateToAggregate is the connectivity state used only for state diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go index 738fba1bc7..16517579f8 100644 --- a/xds/balancer/clustermanager/clustermanager.go +++ b/xds/balancer/clustermanager/clustermanager.go @@ -22,17 +22,25 @@ package clustermanager import ( "encoding/json" "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const balancerName = "xds_cluster_manager_experimental" func init() { diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go index f04d78ed51..7657473966 100644 --- a/xds/balancer/clustermanager/config.go +++ b/xds/balancer/clustermanager/config.go @@ -20,11 +20,16 @@ package clustermanager import ( "encoding/json" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + type childConfig struct { // ChildPolicy is the child policy and it's config. ChildPolicy *internalserviceconfig.BalancerConfig diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go index 015cd2b7af..a8ead24c5b 100644 --- a/xds/balancer/clustermanager/picker.go +++ b/xds/balancer/clustermanager/picker.go @@ -20,9 +20,13 @@ package clustermanager import ( "context" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go index 45b1f6b51c..a871cd36e9 100644 --- a/xds/balancer/clusterresolver/clusterresolver.go +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -23,7 +23,22 @@ import ( "encoding/json" "errors" "fmt" +) + +import ( + "google.golang.org/grpc/attributes" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -31,12 +46,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) // Name is the name of the cluster_resolver balancer. diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go index 5aa1594104..dd33636459 100644 --- a/xds/balancer/clusterresolver/config.go +++ b/xds/balancer/clusterresolver/config.go @@ -22,13 +22,19 @@ import ( "encoding/json" "fmt" "strings" +) - "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // DiscoveryMechanismType is the type of discovery mechanism. type DiscoveryMechanismType int diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go index c135e95152..f9a031a5eb 100644 --- a/xds/balancer/clusterresolver/configbuilder.go +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -22,17 +22,23 @@ import ( "encoding/json" "fmt" "sort" +) + +import ( + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/balancer/weightedtarget" + + "google.golang.org/grpc/resolver" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/balancer/weightedroundrobin" - "google.golang.org/grpc/balancer/weightedtarget" - "google.golang.org/grpc/resolver" ) const million = 1000000 diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go index b30739bb9c..8809bcac92 100644 --- a/xds/balancer/clusterresolver/logging.go +++ b/xds/balancer/clusterresolver/logging.go @@ -20,11 +20,16 @@ package clusterresolver import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-cluster-resolver-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go index d4dbc321da..7ab8e75e38 100644 --- a/xds/balancer/clusterresolver/resource_resolver.go +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -20,7 +20,9 @@ package clusterresolver import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go index 7a639f51a5..2d3553d121 100644 --- a/xds/balancer/clusterresolver/resource_resolver_dns.go +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -20,8 +20,11 @@ package clusterresolver import ( "fmt" +) +import ( "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go index 219a857bd3..1c221d95b6 100644 --- a/xds/balancer/clusterresolver/weightedtarget_config.go +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -19,11 +19,17 @@ package clusterresolver import ( - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" "encoding/json" +) + +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // Target represents one target with the weight and the child policy. type Target struct { // Weight is the weight of the child policy. diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go index 8133d6d4af..4da720fb74 100644 --- a/xds/balancer/loadstore/load_store_wrapper.go +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -21,7 +21,9 @@ package loadstore import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/load" ) diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go index bd32992986..0a635cebbd 100644 --- a/xds/balancer/orca/orca.go +++ b/xds/balancer/orca/orca.go @@ -18,13 +18,19 @@ package orca import ( - "dubbo.apache.org/dubbo-go/v3/xds/utils/balancerload" orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancerload" +) + const mdKey = "X-Endpoint-Load-Metrics-Bin" var logger = grpclog.Component("xds") diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go index f2111fd344..977078fe5f 100644 --- a/xds/balancer/priority/balancer.go +++ b/xds/balancer/priority/balancer.go @@ -28,16 +28,23 @@ import ( "fmt" "sync" "time" +) + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" + + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) // Name is the name of the priority balancer. diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go index 600705da01..9bd4463300 100644 --- a/xds/balancer/priority/balancer_child.go +++ b/xds/balancer/priority/balancer_child.go @@ -21,8 +21,11 @@ package priority import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go index 37cd445604..36b823d41e 100644 --- a/xds/balancer/priority/balancer_priority.go +++ b/xds/balancer/priority/balancer_priority.go @@ -21,9 +21,12 @@ package priority import ( "errors" "time" +) +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" ) diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go index cb337c36e5..0519465124 100644 --- a/xds/balancer/priority/config.go +++ b/xds/balancer/priority/config.go @@ -21,11 +21,16 @@ package priority import ( "encoding/json" "fmt" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // Child is a child of priority balancer. type Child struct { Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"` diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go index 67045bf11e..1cee0c728b 100644 --- a/xds/balancer/priority/config_test.go +++ b/xds/balancer/priority/config_test.go @@ -20,12 +20,18 @@ package priority import ( "testing" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer/roundrobin" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + func TestParseConfig(t *testing.T) { tests := []struct { name string diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go index 9a9f477726..a4eaa36271 100644 --- a/xds/balancer/priority/ignore_resolve_now.go +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -20,8 +20,11 @@ package priority import ( "sync/atomic" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go index 53d2af57be..2e7874c23b 100644 --- a/xds/balancer/priority/logging.go +++ b/xds/balancer/priority/logging.go @@ -20,11 +20,16 @@ package priority import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[priority-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go index c80a89b080..947532d92e 100644 --- a/xds/balancer/priority/utils_test.go +++ b/xds/balancer/priority/utils_test.go @@ -18,7 +18,9 @@ package priority -import "testing" +import ( + "testing" +) func TestCompareStringSlice(t *testing.T) { tests := []struct { diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go index 5cb4aab3d9..71625a0400 100644 --- a/xds/balancer/ringhash/config.go +++ b/xds/balancer/ringhash/config.go @@ -21,7 +21,9 @@ package ringhash import ( "encoding/json" "fmt" +) +import ( "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go index a2a966dc31..ab703c8d8e 100644 --- a/xds/balancer/ringhash/config_test.go +++ b/xds/balancer/ringhash/config_test.go @@ -20,7 +20,9 @@ package ringhash import ( "testing" +) +import ( "github.com/google/go-cmp/cmp" ) diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go index 584f99fe8c..2ed8ea8774 100644 --- a/xds/balancer/ringhash/logging.go +++ b/xds/balancer/ringhash/logging.go @@ -20,11 +20,16 @@ package ringhash import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[ring-hash-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go index 5e0fab0365..c7bd9a8154 100644 --- a/xds/balancer/ringhash/picker.go +++ b/xds/balancer/ringhash/picker.go @@ -20,14 +20,22 @@ package ringhash import ( "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + type picker struct { ring *ring logger *grpclog.PrefixLogger diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go index 68e844cfb4..f0db1e6525 100644 --- a/xds/balancer/ringhash/ring.go +++ b/xds/balancer/ringhash/ring.go @@ -23,8 +23,11 @@ import ( "math" "sort" "strconv" +) +import ( xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go index 2d664e05bb..abf23bbf25 100644 --- a/xds/balancer/ringhash/ring_test.go +++ b/xds/balancer/ringhash/ring_test.go @@ -22,8 +22,11 @@ import ( "fmt" "math" "testing" +) +import ( xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go index 947a20ae20..4001f7737b 100644 --- a/xds/balancer/ringhash/ringhash.go +++ b/xds/balancer/ringhash/ringhash.go @@ -24,17 +24,25 @@ import ( "errors" "fmt" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // Name is the name of the ring_hash balancer. const Name = "ring_hash_experimental" diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go index 92bb3ae5b7..05ad2c9555 100644 --- a/xds/balancer/ringhash/util.go +++ b/xds/balancer/ringhash/util.go @@ -18,7 +18,9 @@ package ringhash -import "context" +import ( + "context" +) type clusterKey struct{} diff --git a/xds/client/attributes.go b/xds/client/attributes.go index ecd3a13dd2..ae2c195a37 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -17,11 +17,16 @@ package client +import ( + _struct "github.com/golang/protobuf/ptypes/struct" + + "google.golang.org/grpc/resolver" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "google.golang.org/grpc/resolver" ) type clientKeyType string @@ -45,6 +50,8 @@ type XDSClient interface { BootstrapConfig() *bootstrap.Config Close() + + SetMetadata(*_struct.Struct) error } // FromResolverState returns the Client from state, or nil if not present. diff --git a/xds/client/authority.go b/xds/client/authority.go index 4cb486eb06..5be5d5b1eb 100644 --- a/xds/client/authority.go +++ b/xds/client/authority.go @@ -20,7 +20,13 @@ package client import ( "errors" "fmt" +) + +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" @@ -151,6 +157,10 @@ type authority struct { refCount int } +func (a *authority) SetMetadata(m *_struct.Struct) error { + return a.controller.SetMetadata(m) +} + // caller must hold parent's authorityMu. func (a *authority) ref() { a.refCount++ diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 1b56d17996..3e87ba198f 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -22,24 +22,30 @@ package bootstrap import ( "bytes" - internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" "encoding/json" "fmt" "io/ioutil" "strings" +) + +import ( + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" +) - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) const ( diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 05b982703f..784b94a0d3 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -19,26 +19,33 @@ package bootstrap import ( - internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" "encoding/json" "errors" "fmt" "os" "testing" +) + +import ( + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "github.com/golang/protobuf/proto" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" +) - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - structpb "github.com/golang/protobuf/ptypes/struct" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) var ( diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go index 47bdbb3284..3ed8b8dcef 100644 --- a/xds/client/bootstrap/logging.go +++ b/xds/client/bootstrap/logging.go @@ -19,10 +19,13 @@ package bootstrap import ( - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-bootstrap] " var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go index bc12eb4299..9751e79a43 100644 --- a/xds/client/bootstrap/template_test.go +++ b/xds/client/bootstrap/template_test.go @@ -17,7 +17,9 @@ package bootstrap -import "testing" +import ( + "testing" +) func Test_percentEncode(t *testing.T) { tests := []struct { diff --git a/xds/client/client.go b/xds/client/client.go index 50d1781c4a..3d8e502f99 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -24,7 +24,13 @@ import ( "fmt" "sync" "time" +) +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" @@ -40,8 +46,9 @@ import ( // style of ccBalancerWrapper so that the Client type does not implement these // exported methods. type clientImpl struct { - done *grpcsync.Event - config *bootstrap.Config + done *grpcsync.Event + config *bootstrap.Config + refreshMetadataCancel func() // authorityMu protects the authority fields. It's necessary because an // authority is created when it's used. @@ -91,6 +98,18 @@ func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration, i return c, nil } +func (c *clientImpl) SetMetadata(m *_struct.Struct) error { + a, _, err := c.findAuthority(resource.ParseName("")) + if err != nil { + return err + } + if err := a.SetMetadata(m); err != nil { + return err + } + a.watchEndpoints("", func(update resource.EndpointsUpdate, err error) {}) + return nil +} + // BootstrapConfig returns the configuration read from the bootstrap file. // Callers must treat the return value as read-only. func (c *clientRefCounted) BootstrapConfig() *bootstrap.Config { diff --git a/xds/client/controller.go b/xds/client/controller.go index ad7b31cd4d..9f53a3b738 100644 --- a/xds/client/controller.go +++ b/xds/client/controller.go @@ -17,6 +17,10 @@ package client +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/controller" @@ -30,6 +34,7 @@ type controllerInterface interface { AddWatch(resourceType resource.ResourceType, resourceName string) RemoveWatch(resourceType resource.ResourceType, resourceName string) ReportLoad(server string) (*load.Store, func()) + SetMetadata(m *_struct.Struct) error Close() } diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go index da5362784f..78743722e3 100644 --- a/xds/client/controller/controller.go +++ b/xds/client/controller/controller.go @@ -29,7 +29,18 @@ import ( "fmt" "sync" "time" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + _struct "github.com/golang/protobuf/ptypes/struct" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" @@ -37,8 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" ) // Controller manages the connection and stream to the control plane. @@ -55,7 +64,7 @@ type Controller struct { logger *grpclog.PrefixLogger cc *grpc.ClientConn // Connection to the management server. - vClient version.VersionedClient + vClient version.MetadataWrappedVersionClient stopRunGoroutine context.CancelFunc backoff func(int) time.Duration @@ -154,6 +163,41 @@ func New(config *bootstrap.ServerConfig, updateHandler pubsub.UpdateHandler, val return ret, nil } +func (t *Controller) SetMetadata(m *_struct.Struct) error { + dopts := []grpc.DialOption{ + t.config.Creds, + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 5 * time.Minute, + Timeout: 20 * time.Second, + }), + } + + cc, err := grpc.Dial(t.config.ServerURI, dopts...) + if err != nil { + // An error from a non-blocking dial indicates something serious. + return fmt.Errorf("xds: failed to dial control plane {%s}: %v", t.config.ServerURI, err) + } + t.cc = cc + builder := version.GetAPIClientBuilder(t.config.TransportAPI) + if builder == nil { + return fmt.Errorf("no client builder for xDS API version: %v", t.config.TransportAPI) + } + v3PBNode, ok := t.config.NodeProto.(*v3corepb.Node) + if ok { + v3PBNode.Metadata = m + } + apiClient, err := builder(version.BuildOptions{NodeProto: t.config.NodeProto, Logger: t.logger}) + if err != nil { + return err + } + t.vClient = apiClient + t.stopRunGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + t.stopRunGoroutine = cancel + go t.run(ctx) + return nil +} + // Close closes the controller. func (t *Controller) Close() { // Note that Close needs to check for nils even if some of them are always diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go index 5408cb123b..52d82fb320 100644 --- a/xds/client/controller/loadreport.go +++ b/xds/client/controller/loadreport.go @@ -19,10 +19,15 @@ package controller import ( "context" +) +import ( + "google.golang.org/grpc" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "google.golang.org/grpc" ) // ReportLoad starts an load reporting stream to the given server. If the server diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go index 0caed731e5..3953dfa20e 100644 --- a/xds/client/controller/transport.go +++ b/xds/client/controller/transport.go @@ -22,13 +22,19 @@ import ( "context" "fmt" "time" +) + +import ( + "github.com/golang/protobuf/proto" + + "google.golang.org/grpc" +) +import ( controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" ) // AddWatch adds a watch for an xDS resource given its type and name. diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go index df3fd610b7..d67e8b80ab 100644 --- a/xds/client/controller/version/v2/client.go +++ b/xds/client/controller/version/v2/client.go @@ -22,21 +22,30 @@ package v2 import ( "context" "fmt" +) + +import ( + v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" - controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + + statuspb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" - statuspb "google.golang.org/genproto/googleapis/rpc/status" +import ( + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) func init() { @@ -52,7 +61,7 @@ var ( } ) -func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { +func newClient(opts controllerversion.BuildOptions) (controllerversion.MetadataWrappedVersionClient, error) { nodeProto, ok := opts.NodeProto.(*v2corepb.Node) if !ok { return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, (*v2corepb.Node)(nil)) @@ -71,6 +80,11 @@ type client struct { logger *grpclog.PrefixLogger } +// SetMetadata update client metadata +func (v2c *client) SetMetadata(p *_struct.Struct) { + v2c.nodeProto.Metadata = p +} + func (v2c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { return v2adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) } diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go index 69d7b5df80..60cd6b3a05 100644 --- a/xds/client/controller/version/v2/loadreport.go +++ b/xds/client/controller/version/v2/loadreport.go @@ -23,20 +23,26 @@ import ( "errors" "fmt" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go index 9fa804febc..ec15a7b711 100644 --- a/xds/client/controller/version/v3/client.go +++ b/xds/client/controller/version/v3/client.go @@ -22,21 +22,30 @@ package v3 import ( "context" "fmt" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + statuspb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +import ( + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) func init() { @@ -52,7 +61,7 @@ var ( } ) -func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { +func newClient(opts controllerversion.BuildOptions) (controllerversion.MetadataWrappedVersionClient, error) { nodeProto, ok := opts.NodeProto.(*v3corepb.Node) if !ok { return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v3corepb.Node{}) @@ -77,6 +86,11 @@ func (v3c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.Cli return v3adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) } +// SetMetadata update client metadata +func (v3c *client) SetMetadata(p *_struct.Struct) { + v3c.nodeProto.Metadata = p +} + // SendRequest sends out a DiscoveryRequest for the given resourceNames, of type // rType, on the provided stream. // diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go index 6a55f9ab46..800b10d2c1 100644 --- a/xds/client/controller/version/v3/loadreport.go +++ b/xds/client/controller/version/v3/loadreport.go @@ -23,20 +23,26 @@ import ( "errors" "fmt" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go index 64f4f4dc19..439b9bb93c 100644 --- a/xds/client/controller/version/version.go +++ b/xds/client/controller/version/version.go @@ -21,18 +21,26 @@ package version import ( "context" "time" +) + +import ( + "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + "google.golang.org/grpc" + + "google.golang.org/protobuf/types/known/anypb" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/anypb" ) var ( - m = make(map[version.TransportAPI]func(opts BuildOptions) (VersionedClient, error)) + m = make(map[version.TransportAPI]func(opts BuildOptions) (MetadataWrappedVersionClient, error)) ) // RegisterAPIClientBuilder registers a client builder for xDS transport protocol @@ -41,13 +49,13 @@ var ( // NOTE: this function must only be called during initialization time (i.e. in // an init() function), and is not thread-safe. If multiple builders are // registered for the same version, the one registered last will take effect. -func RegisterAPIClientBuilder(v version.TransportAPI, f func(opts BuildOptions) (VersionedClient, error)) { +func RegisterAPIClientBuilder(v version.TransportAPI, f func(opts BuildOptions) (MetadataWrappedVersionClient, error)) { m[v] = f } // GetAPIClientBuilder returns the client builder registered for the provided // xDS transport API version. -func GetAPIClientBuilder(version version.TransportAPI) func(opts BuildOptions) (VersionedClient, error) { +func GetAPIClientBuilder(version version.TransportAPI) func(opts BuildOptions) (MetadataWrappedVersionClient, error) { if f, ok := m[version]; ok { return f } @@ -121,3 +129,8 @@ type VersionedClient interface { // invoked. SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error } + +type MetadataWrappedVersionClient interface { + VersionedClient + SetMetadata(p *_struct.Struct) +} diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go index 46568591f9..9cb641a4c0 100644 --- a/xds/client/load/store_test.go +++ b/xds/client/load/store_test.go @@ -22,7 +22,9 @@ import ( "sort" "sync" "testing" +) +import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" ) diff --git a/xds/client/logging.go b/xds/client/logging.go index 2b803f3c70..172d5858aa 100644 --- a/xds/client/logging.go +++ b/xds/client/logging.go @@ -20,11 +20,16 @@ package client import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-client %p] " var logger = grpclog.Component("xds") diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go index 7a8b82f425..490bde0709 100644 --- a/xds/client/pubsub/dump.go +++ b/xds/client/pubsub/dump.go @@ -18,10 +18,13 @@ package pubsub import ( - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" anypb "github.com/golang/protobuf/ptypes/any" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + func rawFromCache(s string, cache interface{}) *anypb.Any { switch c := cache.(type) { case map[string]resource.ListenerUpdate: diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go index 4226f581cf..5875a59525 100644 --- a/xds/client/pubsub/interface.go +++ b/xds/client/pubsub/interface.go @@ -17,7 +17,9 @@ package pubsub -import "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) // UpdateHandler receives and processes (by taking appropriate actions) xDS // resource updates from an APIClient for a specific version. diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go index d4412fd4ac..c14a1dcbbe 100644 --- a/xds/client/pubsub/pubsub.go +++ b/xds/client/pubsub/pubsub.go @@ -25,7 +25,9 @@ package pubsub import ( "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 42f8a7bd57..6765f65e9e 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -17,10 +17,13 @@ package pubsub +import ( + "google.golang.org/protobuf/proto" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/protobuf/proto" ) type watcherInfoWithUpdate struct { @@ -59,6 +62,9 @@ func (pb *Pubsub) callCallback(wiu *watcherInfoWithUpdate) { ccb = func() { wiu.wi.rdsCallback(wiu.update.(resource.RouteConfigUpdate), wiu.err) } } case resource.ClusterResource: + if s, ok := pb.cdsWatchers["*"]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.cdsCallback(wiu.update.(resource.ClusterUpdate), wiu.err) } + } if s, ok := pb.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { ccb = func() { wiu.wi.cdsCallback(wiu.update.(resource.ClusterUpdate), wiu.err) } } @@ -186,7 +192,11 @@ func (pb *Pubsub) NewClusters(updates map[string]resource.ClusterUpdateErrTuple, defer pb.mu.Unlock() for name, uErr := range updates { - if s, ok := pb.cdsWatchers[name]; ok { + s, ok := pb.cdsWatchers[name] + if !ok { + s, ok = pb.cdsWatchers["*"] + } + if ok { if uErr.Err != nil { // On error, keep previous version for each resource. But update // status and error. diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go index 13cd8ed885..120e2231a5 100644 --- a/xds/client/pubsub/watch.go +++ b/xds/client/pubsub/watch.go @@ -21,7 +21,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) @@ -184,6 +186,10 @@ func (pb *Pubsub) watch(wi *watchInfo) (first bool, cancel func() bool) { wi.newUpdate(v) } case resource.ClusterResource: + if v, ok := pb.cdsCache["*"]; ok { + pb.logger.Debugf("CDS resource with name * found in cache: %+v", pretty.ToJSON(v)) + wi.newUpdate(v) + } if v, ok := pb.cdsCache[resourceName]; ok { pb.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) wi.newUpdate(v) diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go index be3269290e..23643bf1e4 100644 --- a/xds/client/resource/errors.go +++ b/xds/client/resource/errors.go @@ -18,7 +18,9 @@ package resource -import "fmt" +import ( + "fmt" +) // ErrorType is the type of the error that the watcher will receive from the xds // client. diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go index ccc786233e..c270aba0fa 100644 --- a/xds/client/resource/filter_chain.go +++ b/xds/client/resource/filter_chain.go @@ -18,22 +18,28 @@ package resource import ( - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" "errors" "fmt" "net" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +import ( v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +) + const ( // Used as the map key for unspecified prefixes. The actual value of this // key is immaterial. diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go index 9e720da1d5..56eff7b437 100644 --- a/xds/client/resource/locality_id.go +++ b/xds/client/resource/locality_id.go @@ -22,7 +22,9 @@ package resource import ( "encoding/json" "fmt" +) +import ( "google.golang.org/grpc/resolver" ) diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go index 163b3035b9..93ccda43f1 100644 --- a/xds/client/resource/matcher.go +++ b/xds/client/resource/matcher.go @@ -20,12 +20,17 @@ package resource import ( "fmt" "strings" +) +import ( + "google.golang.org/grpc/metadata" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" metedatautils "dubbo.apache.org/dubbo-go/v3/xds/utils/metadata" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" - "google.golang.org/grpc/metadata" ) // RouteToMatcher converts a route to a Matcher to match incoming RPC's against. diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go index 86d7253999..9310d3b7bf 100644 --- a/xds/client/resource/matcher_path.go +++ b/xds/client/resource/matcher_path.go @@ -20,7 +20,9 @@ package resource import ( "regexp" "strings" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" ) diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go index 629e8b2397..48a636e62a 100644 --- a/xds/client/resource/name.go +++ b/xds/client/resource/name.go @@ -21,7 +21,9 @@ import ( "net/url" "sort" "strings" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go index 262388af8f..7dff2dd430 100644 --- a/xds/client/resource/type.go +++ b/xds/client/resource/type.go @@ -19,11 +19,16 @@ package resource import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +) + // UpdateValidatorFunc performs validations on update structs using // context/logic available at the xdsClient layer. Since these validation are // performed on internal update structs, they can be shared between different diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go index a5fc9fc469..4d101817aa 100644 --- a/xds/client/resource/type_cds.go +++ b/xds/client/resource/type_cds.go @@ -17,7 +17,9 @@ package resource -import "google.golang.org/protobuf/types/known/anypb" +import ( + "google.golang.org/protobuf/types/known/anypb" +) // ClusterType is the type of cluster from a received CDS response. type ClusterType int diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go index fb5f3af56c..83e63505d5 100644 --- a/xds/client/resource/type_lds.go +++ b/xds/client/resource/type_lds.go @@ -19,11 +19,16 @@ package resource import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" +) + // ListenerUpdate contains information received in an LDS response, which is of // interest to the registered LDS watcher. type ListenerUpdate struct { diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go index 75efd46fcb..9b9b7ca806 100644 --- a/xds/client/resource/type_rds.go +++ b/xds/client/resource/type_rds.go @@ -20,12 +20,18 @@ package resource import ( "regexp" "time" +) + +import ( + "google.golang.org/grpc/codes" + + "google.golang.org/protobuf/types/known/anypb" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" - "google.golang.org/grpc/codes" - "google.golang.org/protobuf/types/known/anypb" ) // RouteConfigUpdate contains information received in an RDS response, which is diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go index c19f6e2ff1..29c09bbe3e 100644 --- a/xds/client/resource/unmarshal.go +++ b/xds/client/resource/unmarshal.go @@ -24,11 +24,16 @@ import ( "fmt" "strings" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + // UnmarshalOptions wraps the input parameters for `UnmarshalXxx` functions. type UnmarshalOptions struct { // Version is the version of the received response. diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go index e1dfc7934d..ef6e753532 100644 --- a/xds/client/resource/unmarshal_cds.go +++ b/xds/client/resource/unmarshal_cds.go @@ -22,20 +22,27 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // TransportSocket proto message has a `name` field which is expected to be set // to this value by the management server. const transportSocketName = "envoy.transport_sockets.tls" @@ -81,6 +88,8 @@ const ( func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { var lbPolicy *ClusterLBPolicyRingHash + // todo @(laurence) this direct set + cluster.LbPolicy = v3clusterpb.Cluster_ROUND_ROBIN switch cluster.GetLbPolicy() { case v3clusterpb.Cluster_ROUND_ROBIN: lbPolicy = nil // The default is round_robin, and there's no config to set. @@ -134,6 +143,10 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu } // Validate and set cluster type from the response. + // todo @laurence this set cluster + if x, ok := cluster.GetClusterDiscoveryType().(*v3clusterpb.Cluster_Type); ok { + x.Type = v3clusterpb.Cluster_EDS + } switch { case cluster.GetType() == v3clusterpb.Cluster_EDS: if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil { diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go index f4403ca371..79a18f65a0 100644 --- a/xds/client/resource/unmarshal_eds.go +++ b/xds/client/resource/unmarshal_eds.go @@ -21,16 +21,23 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalEndpoints processes resources received in an EDS response, // validates them, and transforms them into a native struct which contains only // fields we are interested in. diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index 5e2e246b52..06d1d035a3 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -21,21 +21,30 @@ import ( "errors" "fmt" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v1udpatypepb "github.com/cncf/udpa/go/udpa/type/v1" + v3cncftypepb "github.com/cncf/xds/go/xds/type/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalListener processes resources received in an LDS response, validates // them, and transforms them into a native struct which contains only fields we // are interested in. diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go index e8063a6bee..9868bc46b4 100644 --- a/xds/client/resource/unmarshal_rds.go +++ b/xds/client/resource/unmarshal_rds.go @@ -22,19 +22,27 @@ import ( "regexp" "strings" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalRouteConfig processes resources received in an RDS response, // validates them, and transforms them into a native struct which contains only // fields we are interested in. The provided hostname determines the route diff --git a/xds/client/singleton.go b/xds/client/singleton.go index 0a7ee3ca19..66daecfbec 100644 --- a/xds/client/singleton.go +++ b/xds/client/singleton.go @@ -24,7 +24,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" ) diff --git a/xds/csds/csds.go b/xds/csds/csds.go index 6bbb8e981f..90435de88a 100644 --- a/xds/csds/csds.go +++ b/xds/csds/csds.go @@ -26,22 +26,31 @@ package csds import ( "context" "io" +) - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" // Register v2 xds_client. _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" // Register v3 xds_client. + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) var ( diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go index f6f903824b..90352b4209 100644 --- a/xds/httpfilter/fault/fault.go +++ b/xds/httpfilter/fault/fault.go @@ -27,20 +27,29 @@ import ( "strconv" "sync/atomic" "time" +) + +import ( + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) - cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" - fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" - tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) const headerAbortHTTPStatus = "x-envoy-fault-abort-request" diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go index 4f167590cd..7302f83921 100644 --- a/xds/httpfilter/httpfilter.go +++ b/xds/httpfilter/httpfilter.go @@ -21,10 +21,13 @@ package httpfilter import ( - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" ) +import ( + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +) + // FilterConfig represents an opaque data structure holding configuration for a // filter. Embed this interface to implement it. type FilterConfig interface { diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go index eacf7361c6..a54e572103 100644 --- a/xds/httpfilter/rbac/rbac.go +++ b/xds/httpfilter/rbac/rbac.go @@ -24,17 +24,23 @@ import ( "errors" "fmt" "strings" +) + +import ( + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/rbac" - "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) - v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" - rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/rbac" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) func init() { diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go index 78a433a696..6a48af6dd2 100644 --- a/xds/httpfilter/router/router.go +++ b/xds/httpfilter/router/router.go @@ -21,14 +21,20 @@ package router import ( "fmt" +) + +import ( + pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) - pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) // TypeURL is the message type for the Router configuration. diff --git a/xds/internal/internal.go b/xds/internal/internal.go index 1b596bf357..b778deb60f 100644 --- a/xds/internal/internal.go +++ b/xds/internal/internal.go @@ -23,8 +23,11 @@ package internal import ( "context" "time" +) +import ( "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go index 1c3a947b0f..7a3b9bc081 100644 --- a/xds/resolver/logging.go +++ b/xds/resolver/logging.go @@ -20,11 +20,16 @@ package resolver import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-resolver %p] " var logger = grpclog.Component("xds") diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go index 4640662c0d..5122bf444b 100644 --- a/xds/resolver/serviceconfig.go +++ b/xds/resolver/serviceconfig.go @@ -26,7 +26,19 @@ import ( "strings" "sync/atomic" "time" +) + +import ( + xxhash "github.com/cespare/xxhash/v2" + + "google.golang.org/grpc/codes" + + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -37,10 +49,6 @@ import ( iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" - xxhash "github.com/cespare/xxhash/v2" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" ) const ( diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go index 905aef7390..acb27b10d8 100644 --- a/xds/resolver/watch_service.go +++ b/xds/resolver/watch_service.go @@ -22,7 +22,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go index 0627826dc2..f444d82e82 100644 --- a/xds/resolver/xds_resolver.go +++ b/xds/resolver/xds_resolver.go @@ -23,7 +23,15 @@ import ( "errors" "fmt" "strings" +) + +import ( + "google.golang.org/grpc/credentials" + + "google.golang.org/grpc/resolver" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -31,8 +39,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/resolver" ) const xdsScheme = "xds" diff --git a/xds/server.go b/xds/server.go index 331d460991..2977f764c9 100644 --- a/xds/server.go +++ b/xds/server.go @@ -20,15 +20,27 @@ package xds import ( "context" - "dubbo.apache.org/dubbo-go/v3/xds/internal" "errors" "fmt" "net" "sync" +) +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/server" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" @@ -36,13 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" ) const serverPrefix = "[xds-server %p] " diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index 50ba42b19b..47fc8e7d58 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -24,10 +24,15 @@ import ( "net" "sync" "time" +) +import ( + "google.golang.org/grpc/credentials/tls/certprovider" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" - "google.golang.org/grpc/credentials/tls/certprovider" ) // connWrapper is a thin wrapper around a net.Conn returned by Accept(). It diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go index 971bc58cfe..3a7ded1fc1 100644 --- a/xds/server/listener_wrapper.go +++ b/xds/server/listener_wrapper.go @@ -28,16 +28,23 @@ import ( "sync/atomic" "time" "unsafe" +) + +import ( + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/grpclog" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" internalbackoff "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/grpclog" ) var ( diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go index 552ac0e40d..96b891cfed 100644 --- a/xds/server/rds_handler.go +++ b/xds/server/rds_handler.go @@ -20,7 +20,9 @@ package server import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/server_options.go b/xds/server_options.go index 1d46c3adb7..8ebe349d98 100644 --- a/xds/server_options.go +++ b/xds/server_options.go @@ -20,7 +20,9 @@ package xds import ( "net" +) +import ( "google.golang.org/grpc" "google.golang.org/grpc/connectivity" ) diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go index 7cd305252a..9101eb9fe1 100644 --- a/xds/utils/backoff/backoff.go +++ b/xds/utils/backoff/backoff.go @@ -24,11 +24,16 @@ package backoff import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" +import ( grpcbackoff "google.golang.org/grpc/backoff" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" +) + // Strategy defines the methodology for backing off after a grpc connection // failure. type Strategy interface { diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go index 950eaaa027..1a66fa9293 100644 --- a/xds/utils/balancer/stub/stub.go +++ b/xds/utils/balancer/stub/stub.go @@ -19,7 +19,9 @@ // Package stub implements a balancer for testing purposes. package stub -import "google.golang.org/grpc/balancer" +import ( + "google.golang.org/grpc/balancer" +) // BalancerFuncs contains all balancer.Balancer functions with a preceding // *BalancerData parameter for passing additional instance information. Any diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index fed83e3159..1f27a12b52 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -22,14 +22,21 @@ import ( "fmt" "sync" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" +) + // subBalancerWrapper is used to keep the configurations that will be used to start // the underlying balancer. It can be called to start/stop the underlying // balancer. diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go index 9f6a0c1200..f056ab13ce 100644 --- a/xds/utils/buffer/unbounded.go +++ b/xds/utils/buffer/unbounded.go @@ -18,7 +18,9 @@ // Package buffer provides an implementation of an unbounded buffer. package buffer -import "sync" +import ( + "sync" +) // Unbounded is an implementation of an unbounded buffer which does not use // extra goroutines. This is typically used for passing updates from one entity diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index eaca00b818..f51cf05069 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -27,13 +27,20 @@ import ( "fmt" "strings" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +import ( "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + func init() { //internal.GetXDSHandshakeInfoForTesting = GetHandshakeInfo } diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go index 9e0683bb3e..55862c6968 100644 --- a/xds/utils/credentials/xds/handshake_info_test.go +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -24,7 +24,9 @@ import ( "net/url" "regexp" "testing" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" ) diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index ba5874c21e..20eb1bcc8d 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -20,10 +20,13 @@ package grpclog import ( - "dubbo.apache.org/dubbo-go/v3/common/logger" "os" ) +import ( + "dubbo.apache.org/dubbo-go/v3/common/logger" +) + // Logger is the logger used for the non-depth log functions. var Logger = func() logger.Logger { logger.InitLogger(&logger.Config{ diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go index 6f22bd8911..3d0ef900d8 100644 --- a/xds/utils/grpcutil/metadata.go +++ b/xds/utils/grpcutil/metadata.go @@ -20,7 +20,9 @@ package grpcutil import ( "context" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go index 7a092b2b80..410a4fe89e 100644 --- a/xds/utils/grpcutil/regex.go +++ b/xds/utils/grpcutil/regex.go @@ -18,7 +18,9 @@ package grpcutil -import "regexp" +import ( + "regexp" +) // FullMatchWithRegex returns whether the full text matches the regex provided. func FullMatchWithRegex(re *regexp.Regexp, text string) bool { diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go index 1043d5f81d..c17953f821 100644 --- a/xds/utils/hierarchy/hierarchy_test.go +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -20,9 +20,13 @@ package hierarchy import ( "testing" +) +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" ) diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go index 7d9bc0bd76..ead9a67bfd 100644 --- a/xds/utils/matcher/matcher_header.go +++ b/xds/utils/matcher/matcher_header.go @@ -20,12 +20,15 @@ package matcher import ( "fmt" - "google.golang.org/grpc/metadata" "regexp" "strconv" "strings" ) +import ( + "google.golang.org/grpc/metadata" +) + // HeaderMatcher is an interface for header matchers. These are // documented in (EnvoyProxy link here?). These matchers will match on different // aspects of HTTP header name/value pairs. diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go index f567f31982..f8de56fafb 100644 --- a/xds/utils/matcher/matcher_header_test.go +++ b/xds/utils/matcher/matcher_header_test.go @@ -21,7 +21,9 @@ package matcher import ( "regexp" "testing" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go index ad6d4b283b..52c022f731 100644 --- a/xds/utils/matcher/regex.go +++ b/xds/utils/matcher/regex.go @@ -18,7 +18,9 @@ package matcher -import "regexp" +import ( + "regexp" +) // FullMatchWithRegex returns whether the full text matches the regex provided. func FullMatchWithRegex(re *regexp.Regexp, text string) bool { diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go index 9703fd4cac..632064b8d9 100644 --- a/xds/utils/matcher/string_matcher.go +++ b/xds/utils/matcher/string_matcher.go @@ -25,11 +25,16 @@ import ( "fmt" "regexp" "strings" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcutil" +import ( v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcutil" +) + // StringMatcher contains match criteria for matching a string, and is an // internal representation of the `StringMatcher` proto defined at // https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto. diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go index 9528b57e44..d8d4dcfedd 100644 --- a/xds/utils/matcher/string_matcher_test.go +++ b/xds/utils/matcher/string_matcher_test.go @@ -21,8 +21,11 @@ package matcher import ( "regexp" "testing" +) +import ( v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/google/go-cmp/cmp" ) diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go index 9ebb1b22d5..232d92e588 100644 --- a/xds/utils/metadata/metadata.go +++ b/xds/utils/metadata/metadata.go @@ -20,7 +20,9 @@ package metadata import ( "context" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go index 0177af4b51..e51a107388 100644 --- a/xds/utils/pretty/pretty.go +++ b/xds/utils/pretty/pretty.go @@ -23,10 +23,14 @@ import ( "bytes" "encoding/json" "fmt" +) +import ( "github.com/golang/protobuf/jsonpb" protov1 "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/encoding/protojson" + protov2 "google.golang.org/protobuf/proto" ) diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go index edb39ea92b..7aed8f6877 100644 --- a/xds/utils/rbac/matchers.go +++ b/xds/utils/rbac/matchers.go @@ -21,14 +21,19 @@ import ( "fmt" "net" "regexp" +) - internalmatcher "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" v3route_componentspb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" ) +import ( + internalmatcher "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + // matcher is an interface that takes data about incoming RPC's and returns // whether it matches with whatever matcher implements this interface. type matcher interface { diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go index 3ed7895c03..ae3f73b1a4 100644 --- a/xds/utils/rbac/rbac_engine.go +++ b/xds/utils/rbac/rbac_engine.go @@ -27,9 +27,11 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" +import ( v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -39,6 +41,10 @@ import ( "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" +) + const logLevel = 2 var logger = grpclog.Component("rbac") diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go index 26417daad6..5ac81255a3 100644 --- a/xds/utils/resolver/config_selector.go +++ b/xds/utils/resolver/config_selector.go @@ -22,12 +22,18 @@ package resolver import ( "context" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // ConfigSelector controls what configuration to use for every RPC. type ConfigSelector interface { // Selects the configuration for the RPC, or terminates it using the error. diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go index 520d9229e1..56db8e9ff3 100644 --- a/xds/utils/resolver/passthrough/passthrough.go +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -20,7 +20,9 @@ // name without scheme back to gRPC as resolved address. package passthrough -import "google.golang.org/grpc/resolver" +import ( + "google.golang.org/grpc/resolver" +) const scheme = "passthrough" diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go index 10e2ee7222..d5e3af325a 100644 --- a/xds/utils/resolver/unix/unix.go +++ b/xds/utils/resolver/unix/unix.go @@ -21,11 +21,16 @@ package unix import ( "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/transport/networktype" +import ( "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport/networktype" +) + const unixScheme = "unix" const unixAbstractScheme = "unix-abstract" diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go index badbdbf597..f9b6adcc67 100644 --- a/xds/utils/serviceconfig/serviceconfig.go +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -23,10 +23,15 @@ import ( "encoding/json" "fmt" "time" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + externalserviceconfig "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go index 3a725685db..6936bedfce 100644 --- a/xds/utils/serviceconfig/serviceconfig_test.go +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -22,9 +22,13 @@ import ( "encoding/json" "fmt" "testing" +) +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + externalserviceconfig "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go index 61fbf4a5e7..459bf91040 100644 --- a/xds/utils/wrr/random.go +++ b/xds/utils/wrr/random.go @@ -20,7 +20,9 @@ package wrr import ( "fmt" "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" ) diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index b28f376b50..0ba92b0ac7 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -18,6 +18,7 @@ package xds import ( "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" ) From 226d8a9f132d6c12afaa8e02d7a24fb55a371fd8 Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Tue, 22 Mar 2022 20:11:28 +0800 Subject: [PATCH 08/19] Ftr: xds registry support --- common/constant/env.go | 3 + common/constant/key.go | 4 + common/logger/logger.go | 13 +- go.sum | 2 - imports/imports.go | 3 + protocol/dubbo3/dubbo3_invoker.go | 2 +- registry/xds/registry.go | 178 +++++++++++ remoting/xds/client.go | 291 ++++++++++++++++++ remoting/xds/debug.go | 25 ++ test/xds/main.go | 100 ++++++ xds/balancer/balancer.go | 5 +- xds/balancer/cdsbalancer/cdsbalancer.go | 23 +- xds/balancer/cdsbalancer/cluster_handler.go | 2 + xds/balancer/cdsbalancer/logging.go | 7 +- xds/balancer/clusterimpl/clusterimpl.go | 18 +- xds/balancer/clusterimpl/config.go | 7 +- xds/balancer/clusterimpl/config_test.go | 8 +- xds/balancer/clusterimpl/logging.go | 7 +- xds/balancer/clusterimpl/picker.go | 13 +- .../clustermanager/balancerstateaggregator.go | 8 +- xds/balancer/clustermanager/clustermanager.go | 16 +- xds/balancer/clustermanager/config.go | 7 +- xds/balancer/clustermanager/picker.go | 4 + .../clusterresolver/clusterresolver.go | 21 +- xds/balancer/clusterresolver/config.go | 10 +- xds/balancer/clusterresolver/configbuilder.go | 14 +- xds/balancer/clusterresolver/logging.go | 7 +- .../clusterresolver/resource_resolver.go | 2 + .../clusterresolver/resource_resolver_dns.go | 3 + .../clusterresolver/weightedtarget_config.go | 8 +- xds/balancer/loadstore/load_store_wrapper.go | 2 + xds/balancer/orca/orca.go | 8 +- xds/balancer/priority/balancer.go | 13 +- xds/balancer/priority/balancer_child.go | 3 + xds/balancer/priority/balancer_priority.go | 3 + xds/balancer/priority/config.go | 7 +- xds/balancer/priority/config_test.go | 8 +- xds/balancer/priority/ignore_resolve_now.go | 3 + xds/balancer/priority/logging.go | 7 +- xds/balancer/priority/utils_test.go | 4 +- xds/balancer/ringhash/config.go | 2 + xds/balancer/ringhash/config_test.go | 2 + xds/balancer/ringhash/logging.go | 7 +- xds/balancer/ringhash/picker.go | 10 +- xds/balancer/ringhash/ring.go | 3 + xds/balancer/ringhash/ring_test.go | 3 + xds/balancer/ringhash/ringhash.go | 12 +- xds/balancer/ringhash/util.go | 4 +- xds/client/attributes.go | 9 +- xds/client/authority.go | 10 + xds/client/bootstrap/bootstrap.go | 20 +- xds/client/bootstrap/bootstrap_test.go | 21 +- xds/client/bootstrap/logging.go | 5 +- xds/client/bootstrap/template_test.go | 4 +- xds/client/client.go | 25 +- xds/client/controller.go | 5 + xds/client/controller/controller.go | 50 ++- xds/client/controller/loadreport.go | 7 +- xds/client/controller/transport.go | 10 +- xds/client/controller/version/v2/client.go | 34 +- .../controller/version/v2/loadreport.go | 18 +- xds/client/controller/version/v3/client.go | 32 +- .../controller/version/v3/loadreport.go | 18 +- xds/client/controller/version/version.go | 25 +- xds/client/load/store_test.go | 2 + xds/client/logging.go | 7 +- xds/client/pubsub/dump.go | 5 +- xds/client/pubsub/interface.go | 4 +- xds/client/pubsub/pubsub.go | 2 + xds/client/pubsub/update.go | 14 +- xds/client/pubsub/watch.go | 6 + xds/client/resource/errors.go | 4 +- xds/client/resource/filter_chain.go | 16 +- xds/client/resource/locality_id.go | 2 + xds/client/resource/matcher.go | 7 +- xds/client/resource/matcher_path.go | 2 + xds/client/resource/name.go | 2 + xds/client/resource/type.go | 7 +- xds/client/resource/type_cds.go | 4 +- xds/client/resource/type_lds.go | 7 +- xds/client/resource/type_rds.go | 10 +- xds/client/resource/unmarshal.go | 7 +- xds/client/resource/unmarshal_cds.go | 23 +- xds/client/resource/unmarshal_eds.go | 11 +- xds/client/resource/unmarshal_lds.go | 27 +- xds/client/resource/unmarshal_rds.go | 18 +- xds/client/singleton.go | 2 + xds/csds/csds.go | 13 +- xds/httpfilter/fault/fault.go | 21 +- xds/httpfilter/httpfilter.go | 5 +- xds/httpfilter/rbac/rbac.go | 18 +- xds/httpfilter/router/router.go | 12 +- xds/{ => internal}/internal.go | 5 +- xds/resolver/logging.go | 7 +- xds/resolver/serviceconfig.go | 16 +- xds/resolver/watch_service.go | 2 + xds/resolver/xds_resolver.go | 10 +- xds/server.go | 24 +- xds/server/conn_wrapper.go | 7 +- xds/server/listener_wrapper.go | 13 +- xds/server/rds_handler.go | 2 + xds/server_options.go | 2 + xds/utils/backoff/backoff.go | 7 +- xds/utils/balancer/stub/stub.go | 4 +- xds/utils/balancergroup/balancergroup.go | 11 +- xds/utils/buffer/unbounded.go | 4 +- xds/utils/credentials/xds/handshake_info.go | 9 +- .../credentials/xds/handshake_info_test.go | 2 + xds/utils/grpclog/grpclog.go | 19 +- xds/utils/grpclog/prefixLogger.go | 3 - xds/utils/grpcutil/metadata.go | 2 + xds/utils/grpcutil/regex.go | 4 +- xds/utils/hierarchy/hierarchy_test.go | 4 + xds/utils/matcher/matcher_header.go | 5 +- xds/utils/matcher/matcher_header_test.go | 2 + xds/utils/matcher/regex.go | 4 +- xds/utils/matcher/string_matcher.go | 7 +- xds/utils/matcher/string_matcher_test.go | 3 + xds/utils/metadata/metadata.go | 2 + xds/utils/pretty/pretty.go | 4 + xds/utils/rbac/matchers.go | 7 +- xds/utils/rbac/rbac_engine.go | 8 +- xds/utils/resolver/config_selector.go | 8 +- xds/utils/resolver/passthrough/passthrough.go | 4 +- xds/utils/resolver/unix/unix.go | 7 +- xds/utils/serviceconfig/serviceconfig.go | 5 + xds/utils/serviceconfig/serviceconfig_test.go | 4 + xds/utils/wrr/random.go | 2 + xds/utils/xds_cache/timeoutCache.go | 143 +++++++++ xds/xds_handshake_cluster.go | 1 + 130 files changed, 1614 insertions(+), 222 deletions(-) create mode 100644 registry/xds/registry.go create mode 100644 remoting/xds/client.go create mode 100644 remoting/xds/debug.go create mode 100644 test/xds/main.go rename xds/{ => internal}/internal.go (99%) create mode 100644 xds/utils/xds_cache/timeoutCache.go diff --git a/common/constant/env.go b/common/constant/env.go index 9e20a6d520..511b7325c9 100644 --- a/common/constant/env.go +++ b/common/constant/env.go @@ -23,4 +23,7 @@ const ( ConfigFileEnvKey = "DUBBO_GO_CONFIG_PATH" // AppLogConfFile ... AppLogConfFile = "AppLogConfFile" + + PodNameEnvKey = "POD_NAME" + PodNamespaceEnvKey = "POD_NAMESPACE" ) diff --git a/common/constant/key.go b/common/constant/key.go index 0ab70b38c1..0e1a684d3f 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -275,6 +275,10 @@ const ( ZookeeperKey = "zookeeper" ) +const ( + XDSRegistryKey = "xds" +) + const ( EtcdV3Key = "etcdv3" ) diff --git a/common/logger/logger.go b/common/logger/logger.go index 0af1ea861e..167dbf0fa9 100644 --- a/common/logger/logger.go +++ b/common/logger/logger.go @@ -42,6 +42,7 @@ type DubboLogger struct { type Config struct { LumberjackConfig *lumberjack.Logger `yaml:"lumberjack-config"` ZapConfig *zap.Config `yaml:"zap-config"` + CallerSkip int } // Logger is the interface for Logger types @@ -90,8 +91,16 @@ func InitLogger(conf *Config) { config.ZapConfig = conf.ZapConfig } + if conf != nil { + config.CallerSkip = conf.CallerSkip + } + + if config.CallerSkip == 0 { + config.CallerSkip = 1 + } + if conf == nil || conf.LumberjackConfig == nil { - zapLogger, _ = config.ZapConfig.Build(zap.AddCaller(), zap.AddCallerSkip(1)) + zapLogger, _ = config.ZapConfig.Build(zap.AddCaller(), zap.AddCallerSkip(config.CallerSkip)) } else { config.LumberjackConfig = conf.LumberjackConfig zapLogger = initZapLoggerWithSyncer(config) @@ -145,7 +154,7 @@ func initZapLoggerWithSyncer(conf *Config) *zap.Logger { zap.NewAtomicLevelAt(conf.ZapConfig.Level.Level()), ) - return zap.New(core, zap.AddCaller(), zap.AddCallerSkip(1)) + return zap.New(core, zap.AddCaller(), zap.AddCallerSkip(conf.CallerSkip)) } // getEncoder get encoder by config, zapcore support json and console encoder diff --git a/go.sum b/go.sum index 481591584b..dc1bbb3fee 100644 --- a/go.sum +++ b/go.sum @@ -1306,8 +1306,6 @@ google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.44.0 h1:weqSxi/TMs1SqFRMHCtBgXRs8k3X39QIDEZ0pRcttUg= -google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0 h1:NEpgUqV3Z+ZjkqMsxMg11IaDrXY4RY6CQukSGK0uI1M= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/imports/imports.go b/imports/imports.go index 3251ca61df..ab40b61968 100644 --- a/imports/imports.go +++ b/imports/imports.go @@ -72,5 +72,8 @@ import ( _ "dubbo.apache.org/dubbo-go/v3/registry/polaris" _ "dubbo.apache.org/dubbo-go/v3/registry/protocol" _ "dubbo.apache.org/dubbo-go/v3/registry/servicediscovery" + _ "dubbo.apache.org/dubbo-go/v3/registry/xds" _ "dubbo.apache.org/dubbo-go/v3/registry/zookeeper" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" ) diff --git a/protocol/dubbo3/dubbo3_invoker.go b/protocol/dubbo3/dubbo3_invoker.go index 1e33bf2eb0..b87c0de97c 100644 --- a/protocol/dubbo3/dubbo3_invoker.go +++ b/protocol/dubbo3/dubbo3_invoker.go @@ -230,9 +230,9 @@ func (di *DubboInvoker) getTimeout(invocation *invocation_impl.RPCInvocation) ti func (di *DubboInvoker) IsAvailable() bool { client := di.getClient() if client != nil { + // FIXME here can't check if tcp server is started now!!! return client.IsAvailable() } - return false } diff --git a/registry/xds/registry.go b/registry/xds/registry.go new file mode 100644 index 0000000000..b0050da665 --- /dev/null +++ b/registry/xds/registry.go @@ -0,0 +1,178 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + "bytes" + "os" + "strconv" + "strings" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/extension" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/remoting/xds" +) + +var localIP = "" + +const ( + // RegistryConnDelay registry connection delay + RegistryConnDelay = 3 +) + +func init() { + localIP = common.GetLocalIp() + extension.SetRegistry(constant.XDSRegistryKey, newXDSRegistry) +} + +type xdsRegistry struct { + xdsWrappedClient *xds.WrappedClient + registryURL *common.URL +} + +func isProvider(url *common.URL) bool { + return getCategory(url) == "providers" +} + +func getCategory(url *common.URL) string { + role, _ := strconv.Atoi(url.GetParam(constant.RegistryRoleKey, strconv.Itoa(constant.NacosDefaultRoleType))) + category := common.DubboNodes[role] + return category +} + +func getServiceName(url *common.URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(getCategory(url))) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + +func getSubscribeName(url *common.URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(common.DubboNodes[common.PROVIDER])) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + +func appendParam(target *bytes.Buffer, url *common.URL, key string) { + value := url.GetParam(key, "") + target.Write([]byte(constant.NacosServiceNameSeparator)) + if strings.TrimSpace(value) != "" { + target.Write([]byte(value)) + } +} + +// Register will register the service @url to its nacos registry center +func (nr *xdsRegistry) Register(url *common.URL) error { + if !isProvider(url) { + return nil + } + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), true) +} + +// UnRegister +func (nr *xdsRegistry) UnRegister(url *common.URL) error { + return nr.xdsWrappedClient.ChangeInterfaceMap(getServiceName(url), false) +} + +// Subscribe from xds client +func (nr *xdsRegistry) Subscribe(url *common.URL, notifyListener registry.NotifyListener) error { + if isProvider(url) { + return nil + } + hostAddr, svcAddr, err := nr.getHostAddrFromURL(url) + if err != nil { + return err + } + return nr.xdsWrappedClient.Subscribe(svcAddr, url.GetParam(constant.InterfaceKey, ""), hostAddr, notifyListener) +} + +// UnSubscribe from xds client +func (nr *xdsRegistry) UnSubscribe(url *common.URL, _ registry.NotifyListener) error { + _, svcAddr, err := nr.getHostAddrFromURL(url) + if err != nil { + return err + } + nr.xdsWrappedClient.UnSubscribe(svcAddr) + return nil +} + +// GetURL gets its registration URL +func (nr *xdsRegistry) GetURL() *common.URL { + return nr.registryURL +} + +func (nr *xdsRegistry) getHostAddrFromURL(url *common.URL) (string, string, error) { + svcName := getSubscribeName(url) + hostAddr, err := nr.xdsWrappedClient.GetHostAddrFromPilot(svcName) + return hostAddr, svcName, err +} + +// IsAvailable determines nacos registry center whether it is available +func (nr *xdsRegistry) IsAvailable() bool { + // TODO + return true +} + +// nolint +func (nr *xdsRegistry) Destroy() { + // todo unregistry all + //for _, url := range nr.registryUrls { + // err := nr.UnRegister(url) + // logger.Infof("DeRegister Nacos URL:%+v", url) + // if err != nil { + // logger.Errorf("Deregister URL:%+v err:%v", url, err.Error()) + // } + //} + return +} + +// newXDSRegistry will create new instance +func newXDSRegistry(url *common.URL) (registry.Registry, error) { + logger.Infof("[XDS Registry] New XDS registry with url = %+v", url.ToMap()) + pn := os.Getenv(constant.PodNameEnvKey) + ns := os.Getenv(constant.PodNamespaceEnvKey) + if pn == "" || ns == "" { + return nil, perrors.New("POD_NAME and POD_NAMESPACE can't be empty when using xds registry") + } + + wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, url.Ip) + if err != nil { + return nil, err + } + tmpRegistry := &xdsRegistry{ + xdsWrappedClient: wrappedXDSClient, + registryURL: url, + } + return tmpRegistry, nil +} diff --git a/remoting/xds/client.go b/remoting/xds/client.go new file mode 100644 index 0000000000..cc9f93528b --- /dev/null +++ b/remoting/xds/client.go @@ -0,0 +1,291 @@ +package xds + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "strings" + "sync" + "time" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + structpb "github.com/golang/protobuf/ptypes/struct" + + perrors "github.com/pkg/errors" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/remoting" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +) + +const ( + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" +) + +type WrappedClient struct { + interfaceAppNameMap map[string]string + subscribeStopChMap sync.Map + podName string + namespace string + localIP string // to find hostAddr by cds and eds + istiodHostName string // todo: here must be istiod-grpc.istio-system.svc.cluster.local + hostAddr string // dubbo-go-app.default.svc.cluster.local:20000 + istiodPodIP string // to call istiod unexposed debug port 8080 + xdsClient client.XDSClient + lock sync.Mutex + + endpointClusterMap sync.Map +} + +func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*WrappedClient, error) { + // get hostname from http://localhost:8080/debug/endpointz + newClient := &WrappedClient{ + podName: podName, + namespace: namespace, + localIP: localIP, + istiodHostName: istiodHostName, + interfaceAppNameMap: make(map[string]string), + } + if err := newClient.initClientAndloadLocalHostAddr(); err != nil { + return nil, err + } + return newClient, nil +} + +func (w *WrappedClient) getInterfaceHostAddrMapFromPilot() (map[string]string, error) { + req, _ := http.NewRequest(http.MethodGet, "http://"+w.istiodPodIP+":8080/debug/adsz", nil) + token, err := os.ReadFile("/var/run/secrets/token/istio-token") + if err != nil { + return nil, err + } + req.Header.Add("Authorization", "Bearer "+string(token)) + rsp, err := http.DefaultClient.Do(req) + if err != nil { + logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio %s with error %s\n", w.istiodHostName, err) + return nil, err + } + + data, err := ioutil.ReadAll(rsp.Body) + adszRsp := &ADSZResponse{} + if err := json.Unmarshal(data, adszRsp); err != nil { + return nil, err + } + return adszRsp.GetMap(), nil +} + +// GetHostAddrFromPilot todo 1. timeout 2. hostAddr change? +func (w *WrappedClient) GetHostAddrFromPilot(serviceKey string) (string, error) { + for { + if interfaceHostMap, err := w.getInterfaceHostAddrMapFromPilot(); err != nil { + return "", err + } else { + hostName, ok := interfaceHostMap[serviceKey] + if !ok { + logger.Infof("[XDS Wrapped Client] Try getting service %s 's host from istio %d\n", serviceKey, w.istiodHostName) + time.Sleep(time.Millisecond * 100) + continue + } + return hostName, nil + } + } +} + +func (w *WrappedClient) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { + _, ok := w.subscribeStopChMap.Load(svcUniqueName) + if ok { + return perrors.Errorf("XDS WrappedClient subscribe interface %s failed, subscription already exist.", interfaceName) + } + stopCh := make(chan struct{}) + w.subscribeStopChMap.Store(svcUniqueName, stopCh) + ipPort := strings.Split(hostAddr, ":") + hostName := ipPort[0] + port := ipPort[1] + // todo cds, eds + cancel := w.xdsClient.WatchEndpoints(fmt.Sprintf("outbound|%s||%s", port, hostName), func(update resource.EndpointsUpdate, err error) { + for _, v := range update.Localities { + for _, e := range v.Endpoints { + // todo 1. register protocol in server side metadata 2. get metadata from endpoint, for router + url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", e.Address, interfaceName)) + logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", + interfaceName, e.Address, e.HealthStatus) + if e.HealthStatus == resource.EndpointHealthStatusHealthy { + lst.Notify(®istry.ServiceEvent{ + Action: remoting.EventTypeUpdate, + Service: url, + }) + } else { + lst.Notify(®istry.ServiceEvent{ + Action: remoting.EventTypeDel, + Service: url, + }) + } + } + } + }) + <-stopCh + cancel() + return nil +} + +func (w *WrappedClient) UnSubscribe(svcUniqueName string) { + if stopCh, ok := w.subscribeStopChMap.Load(svcUniqueName); ok { + close(stopCh.(chan struct{})) + } + w.subscribeStopChMap.Delete(svcUniqueName) +} + +func (w *WrappedClient) interfaceAppNameMap2String() string { + data, _ := json.Marshal(w.interfaceAppNameMap) + return string(data) +} + +// ChangeInterfaceMap change the map of interfaceName -> appname, if add is true, register, else unregister +func (w *WrappedClient) ChangeInterfaceMap(interfaceName string, add bool) error { + w.lock.Lock() + defer w.lock.Unlock() + if add { + w.interfaceAppNameMap[interfaceName] = w.hostAddr + } else { + delete(w.interfaceAppNameMap, interfaceName) + } + if w.xdsClient == nil { + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + if err != nil { + return err + } + w.xdsClient = xdsClient + return nil + } + + if err := w.xdsClient.SetMetadata(getDubboGoMetadata(w.interfaceAppNameMap2String())); err != nil { + return err + } + return nil +} + +func (w *WrappedClient) initClientAndloadLocalHostAddr() error { + // call watch and refresh istiod debug interface + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + if err != nil { + return err + } + stopCh := make(chan struct{}) + foundLocal := false + foundIstiod := false + cancel := xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + if update.ClusterName == "" { + return + } + clusterNameList := strings.Split(update.ClusterName, "|") + // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! + if clusterNameList[3] == w.istiodHostName { + // 1. find istiod podIP + // todo: When would eds level watch be cancelled? + _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + if foundIstiod { + return + } + for _, v := range endpoint.Localities { + for _, e := range v.Endpoints { + w.endpointClusterMap.Store(e.Address, update.ClusterName) + addrs := strings.Split(e.Address, ":") + w.istiodPodIP = addrs[0] + foundIstiod = true + if foundLocal && foundIstiod { + stopCh <- struct{}{} + } + } + } + }) + return + } + // 2. found local hostAddr + // todo: When would eds level watch be cancelled? + _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + if foundLocal { + return + } + for _, v := range endpoint.Localities { + for _, e := range v.Endpoints { + w.endpointClusterMap.Store(e.Address, update.ClusterName) + addrs := strings.Split(e.Address, ":") + if addrs[0] == w.localIP { + clusterNames := strings.Split(update.ClusterName, "|") + w.hostAddr = clusterNames[3] + ":" + clusterNames[1] + foundLocal = true + } + if foundLocal && foundIstiod { + stopCh <- struct{}{} + } + } + } + }) + }) + <-stopCh + cancel() + w.xdsClient = xdsClient + return nil +} + +func newxdsClient(localIP, podName, namespace, dubboGoMetadata, istiodIP string) (client.XDSClient, error) { + v3NodeProto := &v3corepb.Node{ + Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + } + + nonNilCredsConfigV2 := &bootstrap.Config{ + XDSServer: &bootstrap.ServerConfig{ + ServerURI: istiodIP + ":15010", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + + newClient, err := client.NewWithConfig(nonNilCredsConfigV2) + if err != nil { + return nil, err + } + if err := newClient.SetMetadata(getDubboGoMetadata(dubboGoMetadata)); err != nil { + return nil, err + } + return newClient, nil +} + +func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { + return &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "CLUSTER_ID":{ + Kind: &structpb.Value_StringValue{StringValue: "Kubernetes"}, + }, + "LABELS": { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "DUBBO_GO": { + Kind: &structpb.Value_StringValue{StringValue: dubboGoMetadata}, + }, + }, + }}, + }, + }, + } +} diff --git a/remoting/xds/debug.go b/remoting/xds/debug.go new file mode 100644 index 0000000000..02e23789ad --- /dev/null +++ b/remoting/xds/debug.go @@ -0,0 +1,25 @@ +package xds + +import ( + "encoding/json" +) + +type ADSZResponse struct { + Clients []ADSZClient `json:"clients"` +} + +type ADSZClient struct { + Metadata map[string]interface{} `json:"metadata"` +} + +func (a *ADSZResponse) GetMap() map[string]string { + result := make(map[string]string) + for _, c := range a.Clients { + resultMap := make(map[string]string) + json.Unmarshal([]byte(c.Metadata["LABELS"].(map[string]interface{})["DUBBO_GO"].(string)), &resultMap) + for k, v := range resultMap { + result[k] = v + } + } + return result +} diff --git a/test/xds/main.go b/test/xds/main.go new file mode 100644 index 0000000000..119f71e4e5 --- /dev/null +++ b/test/xds/main.go @@ -0,0 +1,100 @@ +package main + +import ( + "fmt" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + structpb "github.com/golang/protobuf/ptypes/struct" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" + _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +) + +const ( + gRPCUserAgentName = "gRPC Go" + clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" +) + +// ATTENTION! export GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT=false +func main() { + v3NodeProto := &v3corepb.Node{ + Id: "sidecar~172.1.1.1~sleep-55b5877479-rwcct.default~default.svc.cluster.local", + UserAgentName: gRPCUserAgentName, + Cluster: "KUBERNETES", + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + Metadata: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "CLUSTER_ID":{ + Kind: &structpb.Value_StringValue{StringValue: "Kubernetes"}, + }, + "LABELS": { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + "label1": { + Kind: &structpb.Value_StringValue{StringValue: "val1"}, + }, + "label2": { + Kind: &structpb.Value_StringValue{StringValue: "val2"}, + }, + }, + }}, + }, + }, + }, + } + + nonNilCredsConfigV2 := &bootstrap.Config{ + XDSServer: &bootstrap.ServerConfig{ + ServerURI: "localhost:15010", + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + //CredsType: "google_default", + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + + xdsClient, err := client.NewWithConfig(nonNilCredsConfigV2) + if err != nil { + panic(err) + } + + //clusterName := "outbound|20000||dubbo-go-app.default.svc.cluster.local" // + //clusterName := "outbound|8848||nacos.default.svc.cluster.local" + //endpointClusterMap := sync.Map{} + //xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + // xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + // for _, v := range endpoint.Localities { + // for _, e := range v.Endpoints { + // endpointClusterMap.Store(e.Address, update.ClusterName) + // } + // } + // }) + //}) + + // + xdsClient.WatchEndpoints("outbound|15010||istiod.istio-system.svc.cluster.local", func(update resource.EndpointsUpdate, err error) { + fmt.Printf("%+v\n err = %s", update, err) + }) + + //xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + // fmt.Printf("%+v\n err = %s", update, err) + // + //}) + // + + select {} +} diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go index 5ac911301d..490124cab5 100644 --- a/xds/balancer/balancer.go +++ b/xds/balancer/balancer.go @@ -19,11 +19,14 @@ // Package balancer installs all the xds balancers. package balancer +import ( + _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer +) + import ( _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/cdsbalancer" // Register the CDS balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" // Register the xds_cluster_impl balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" // Register the xds_cluster_manager balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" // Register the xds_cluster_resolver balancer _ "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" // Register the priority balancer - _ "google.golang.org/grpc/balancer/weightedtarget" // Register the weighted_target balancer ) diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index 9ab069e1de..ab95d5687f 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -21,7 +21,23 @@ import ( "encoding/json" "errors" "fmt" +) + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/tls/certprovider" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterresolver" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client" @@ -32,13 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/tls/certprovider" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) const ( diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go index 137825309b..dec61af342 100644 --- a/xds/balancer/cdsbalancer/cluster_handler.go +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -19,7 +19,9 @@ package cdsbalancer import ( "errors" "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go index e0d21c0c97..e19b4d0cad 100644 --- a/xds/balancer/cdsbalancer/logging.go +++ b/xds/balancer/cdsbalancer/logging.go @@ -20,11 +20,16 @@ package cdsbalancer import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[cds-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go index 0be70ec12f..c79a866dd7 100644 --- a/xds/balancer/clusterimpl/clusterimpl.go +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -24,12 +24,24 @@ package clusterimpl import ( - internal "dubbo.apache.org/dubbo-go/v3/xds" "encoding/json" "fmt" "sync" "sync/atomic" +) + +import ( + "google.golang.org/grpc/balancer" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( + internal "dubbo.apache.org/dubbo-go/v3/xds" "dubbo.apache.org/dubbo-go/v3/xds/balancer/loadstore" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/load" @@ -38,10 +50,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) const ( diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go index a33ef0472a..9fe1a41a7e 100644 --- a/xds/balancer/clusterimpl/config.go +++ b/xds/balancer/clusterimpl/config.go @@ -20,11 +20,16 @@ package clusterimpl import ( "encoding/json" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // DropConfig contains the category, and drop ratio. type DropConfig struct { Category string diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go index b217a32119..0bdaa66cf9 100644 --- a/xds/balancer/clusterimpl/config_test.go +++ b/xds/balancer/clusterimpl/config_test.go @@ -20,14 +20,20 @@ package clusterimpl import ( "testing" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" _ "google.golang.org/grpc/balancer/roundrobin" _ "google.golang.org/grpc/balancer/weightedtarget" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + const ( testJSONConfig = `{ "cluster": "test_cluster", diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go index 70c8d2dad1..b94aa44256 100644 --- a/xds/balancer/clusterimpl/logging.go +++ b/xds/balancer/clusterimpl/logging.go @@ -20,11 +20,16 @@ package clusterimpl import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-cluster-impl-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go index 5e3da761fd..39874a6f86 100644 --- a/xds/balancer/clusterimpl/picker.go +++ b/xds/balancer/clusterimpl/picker.go @@ -19,16 +19,23 @@ package clusterimpl import ( - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" +) + // NewRandomWRR is used when calculating drops. It's exported so that tests can // override it. var NewRandomWRR = wrr.NewRandom diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go index aec4fb658c..8b0ad174df 100644 --- a/xds/balancer/clustermanager/balancerstateaggregator.go +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -21,13 +21,19 @@ package clustermanager import ( "fmt" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + type subBalancerState struct { state balancer.State // stateToAggregate is the connectivity state used only for state diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go index 738fba1bc7..16517579f8 100644 --- a/xds/balancer/clustermanager/clustermanager.go +++ b/xds/balancer/clustermanager/clustermanager.go @@ -22,17 +22,25 @@ package clustermanager import ( "encoding/json" "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const balancerName = "xds_cluster_manager_experimental" func init() { diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go index f04d78ed51..7657473966 100644 --- a/xds/balancer/clustermanager/config.go +++ b/xds/balancer/clustermanager/config.go @@ -20,11 +20,16 @@ package clustermanager import ( "encoding/json" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + type childConfig struct { // ChildPolicy is the child policy and it's config. ChildPolicy *internalserviceconfig.BalancerConfig diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go index 015cd2b7af..a8ead24c5b 100644 --- a/xds/balancer/clustermanager/picker.go +++ b/xds/balancer/clustermanager/picker.go @@ -20,9 +20,13 @@ package clustermanager import ( "context" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go index 45b1f6b51c..a871cd36e9 100644 --- a/xds/balancer/clusterresolver/clusterresolver.go +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -23,7 +23,22 @@ import ( "encoding/json" "errors" "fmt" +) + +import ( + "google.golang.org/grpc/attributes" + + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/balancer/base" + + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -31,12 +46,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/attributes" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/balancer/base" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) // Name is the name of the cluster_resolver balancer. diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go index 5aa1594104..dd33636459 100644 --- a/xds/balancer/clusterresolver/config.go +++ b/xds/balancer/clusterresolver/config.go @@ -22,13 +22,19 @@ import ( "encoding/json" "fmt" "strings" +) - "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // DiscoveryMechanismType is the type of discovery mechanism. type DiscoveryMechanismType int diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go index c135e95152..f9a031a5eb 100644 --- a/xds/balancer/clusterresolver/configbuilder.go +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -22,17 +22,23 @@ import ( "encoding/json" "fmt" "sort" +) + +import ( + "google.golang.org/grpc/balancer/roundrobin" + "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/balancer/weightedtarget" + + "google.golang.org/grpc/resolver" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clusterimpl" "dubbo.apache.org/dubbo-go/v3/xds/balancer/priority" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" - "google.golang.org/grpc/balancer/roundrobin" - "google.golang.org/grpc/balancer/weightedroundrobin" - "google.golang.org/grpc/balancer/weightedtarget" - "google.golang.org/grpc/resolver" ) const million = 1000000 diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go index b30739bb9c..8809bcac92 100644 --- a/xds/balancer/clusterresolver/logging.go +++ b/xds/balancer/clusterresolver/logging.go @@ -20,11 +20,16 @@ package clusterresolver import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-cluster-resolver-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go index d4dbc321da..7ab8e75e38 100644 --- a/xds/balancer/clusterresolver/resource_resolver.go +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -20,7 +20,9 @@ package clusterresolver import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go index 7a639f51a5..2d3553d121 100644 --- a/xds/balancer/clusterresolver/resource_resolver_dns.go +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -20,8 +20,11 @@ package clusterresolver import ( "fmt" +) +import ( "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go index 219a857bd3..1c221d95b6 100644 --- a/xds/balancer/clusterresolver/weightedtarget_config.go +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -19,11 +19,17 @@ package clusterresolver import ( - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" "encoding/json" +) + +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // Target represents one target with the weight and the child policy. type Target struct { // Weight is the weight of the child policy. diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go index 8133d6d4af..4da720fb74 100644 --- a/xds/balancer/loadstore/load_store_wrapper.go +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -21,7 +21,9 @@ package loadstore import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/load" ) diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go index bd32992986..0a635cebbd 100644 --- a/xds/balancer/orca/orca.go +++ b/xds/balancer/orca/orca.go @@ -18,13 +18,19 @@ package orca import ( - "dubbo.apache.org/dubbo-go/v3/xds/utils/balancerload" orcapb "github.com/cncf/xds/go/xds/data/orca/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/balancerload" +) + const mdKey = "X-Endpoint-Load-Metrics-Bin" var logger = grpclog.Component("xds") diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go index f2111fd344..977078fe5f 100644 --- a/xds/balancer/priority/balancer.go +++ b/xds/balancer/priority/balancer.go @@ -28,16 +28,23 @@ import ( "fmt" "sync" "time" +) + +import ( + "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" + + "google.golang.org/grpc/serviceconfig" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/balancergroup" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/hierarchy" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/grpc/balancer" - "google.golang.org/grpc/resolver" - "google.golang.org/grpc/serviceconfig" ) // Name is the name of the priority balancer. diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go index 600705da01..9bd4463300 100644 --- a/xds/balancer/priority/balancer_child.go +++ b/xds/balancer/priority/balancer_child.go @@ -21,8 +21,11 @@ package priority import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go index 37cd445604..36b823d41e 100644 --- a/xds/balancer/priority/balancer_priority.go +++ b/xds/balancer/priority/balancer_priority.go @@ -21,9 +21,12 @@ package priority import ( "errors" "time" +) +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" + "google.golang.org/grpc/connectivity" ) diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go index cb337c36e5..0519465124 100644 --- a/xds/balancer/priority/config.go +++ b/xds/balancer/priority/config.go @@ -21,11 +21,16 @@ package priority import ( "encoding/json" "fmt" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/serviceconfig" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // Child is a child of priority balancer. type Child struct { Config *internalserviceconfig.BalancerConfig `json:"config,omitempty"` diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go index 67045bf11e..1cee0c728b 100644 --- a/xds/balancer/priority/config_test.go +++ b/xds/balancer/priority/config_test.go @@ -20,12 +20,18 @@ package priority import ( "testing" +) - internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer/roundrobin" ) +import ( + internalserviceconfig "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + func TestParseConfig(t *testing.T) { tests := []struct { name string diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go index 9a9f477726..a4eaa36271 100644 --- a/xds/balancer/priority/ignore_resolve_now.go +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -20,8 +20,11 @@ package priority import ( "sync/atomic" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go index 53d2af57be..2e7874c23b 100644 --- a/xds/balancer/priority/logging.go +++ b/xds/balancer/priority/logging.go @@ -20,11 +20,16 @@ package priority import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[priority-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go index c80a89b080..947532d92e 100644 --- a/xds/balancer/priority/utils_test.go +++ b/xds/balancer/priority/utils_test.go @@ -18,7 +18,9 @@ package priority -import "testing" +import ( + "testing" +) func TestCompareStringSlice(t *testing.T) { tests := []struct { diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go index 5cb4aab3d9..71625a0400 100644 --- a/xds/balancer/ringhash/config.go +++ b/xds/balancer/ringhash/config.go @@ -21,7 +21,9 @@ package ringhash import ( "encoding/json" "fmt" +) +import ( "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go index a2a966dc31..ab703c8d8e 100644 --- a/xds/balancer/ringhash/config_test.go +++ b/xds/balancer/ringhash/config_test.go @@ -20,7 +20,9 @@ package ringhash import ( "testing" +) +import ( "github.com/google/go-cmp/cmp" ) diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go index 584f99fe8c..2ed8ea8774 100644 --- a/xds/balancer/ringhash/logging.go +++ b/xds/balancer/ringhash/logging.go @@ -20,11 +20,16 @@ package ringhash import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[ring-hash-lb %p] " var logger = grpclog.Component("xds") diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go index 5e0fab0365..c7bd9a8154 100644 --- a/xds/balancer/ringhash/picker.go +++ b/xds/balancer/ringhash/picker.go @@ -20,14 +20,22 @@ package ringhash import ( "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + type picker struct { ring *ring logger *grpclog.PrefixLogger diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go index 68e844cfb4..f0db1e6525 100644 --- a/xds/balancer/ringhash/ring.go +++ b/xds/balancer/ringhash/ring.go @@ -23,8 +23,11 @@ import ( "math" "sort" "strconv" +) +import ( xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go index 2d664e05bb..abf23bbf25 100644 --- a/xds/balancer/ringhash/ring_test.go +++ b/xds/balancer/ringhash/ring_test.go @@ -22,8 +22,11 @@ import ( "fmt" "math" "testing" +) +import ( xxhash "github.com/cespare/xxhash/v2" + "google.golang.org/grpc/resolver" ) diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go index 947a20ae20..4001f7737b 100644 --- a/xds/balancer/ringhash/ringhash.go +++ b/xds/balancer/ringhash/ringhash.go @@ -24,17 +24,25 @@ import ( "errors" "fmt" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( "google.golang.org/grpc/balancer" "google.golang.org/grpc/balancer/base" "google.golang.org/grpc/balancer/weightedroundrobin" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" + "google.golang.org/grpc/serviceconfig" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // Name is the name of the ring_hash balancer. const Name = "ring_hash_experimental" diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go index 92bb3ae5b7..05ad2c9555 100644 --- a/xds/balancer/ringhash/util.go +++ b/xds/balancer/ringhash/util.go @@ -18,7 +18,9 @@ package ringhash -import "context" +import ( + "context" +) type clusterKey struct{} diff --git a/xds/client/attributes.go b/xds/client/attributes.go index ecd3a13dd2..ae2c195a37 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -17,11 +17,16 @@ package client +import ( + _struct "github.com/golang/protobuf/ptypes/struct" + + "google.golang.org/grpc/resolver" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "google.golang.org/grpc/resolver" ) type clientKeyType string @@ -45,6 +50,8 @@ type XDSClient interface { BootstrapConfig() *bootstrap.Config Close() + + SetMetadata(*_struct.Struct) error } // FromResolverState returns the Client from state, or nil if not present. diff --git a/xds/client/authority.go b/xds/client/authority.go index 4cb486eb06..5be5d5b1eb 100644 --- a/xds/client/authority.go +++ b/xds/client/authority.go @@ -20,7 +20,13 @@ package client import ( "errors" "fmt" +) + +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" @@ -151,6 +157,10 @@ type authority struct { refCount int } +func (a *authority) SetMetadata(m *_struct.Struct) error { + return a.controller.SetMetadata(m) +} + // caller must hold parent's authorityMu. func (a *authority) ref() { a.refCount++ diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 2b9c734ee7..3e87ba198f 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -26,20 +26,26 @@ import ( "fmt" "io/ioutil" "strings" +) + +import ( + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - internal "dubbo.apache.org/dubbo-go/v3/xds" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/jsonpb" "github.com/golang/protobuf/proto" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" +) - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) const ( @@ -333,7 +339,7 @@ func NewConfigFromContents(data []byte) (*Config, error) { return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err) } configs := make(map[string]*certprovider.BuildableConfig) - getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + getBuilder := internal2.GetCertificateProviderBuilder.(func(string) certprovider.Builder) for instance, data := range providerInstances { var nameAndConfig struct { PluginName string `json:"plugin_name"` diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 37e4b99b44..784b94a0d3 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -24,21 +24,28 @@ import ( "fmt" "os" "testing" +) + +import ( + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - internal "dubbo.apache.org/dubbo-go/v3/xds" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" "github.com/golang/protobuf/proto" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" + "google.golang.org/grpc" "google.golang.org/grpc/credentials/google" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/tls/certprovider" +) - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - structpb "github.com/golang/protobuf/ptypes/struct" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) var ( @@ -647,7 +654,7 @@ func TestNewConfigWithCertificateProviders(t *testing.T) { }`, } - getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder) + getBuilder := internal2.GetCertificateProviderBuilder.(func(string) certprovider.Builder) parser := getBuilder(fakeCertProviderName) if parser == nil { t.Fatalf("missing certprovider plugin %q", fakeCertProviderName) diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go index 47bdbb3284..3ed8b8dcef 100644 --- a/xds/client/bootstrap/logging.go +++ b/xds/client/bootstrap/logging.go @@ -19,10 +19,13 @@ package bootstrap import ( - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-bootstrap] " var logger = internalgrpclog.NewPrefixLogger(grpclog.Component("xds"), prefix) diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go index bc12eb4299..9751e79a43 100644 --- a/xds/client/bootstrap/template_test.go +++ b/xds/client/bootstrap/template_test.go @@ -17,7 +17,9 @@ package bootstrap -import "testing" +import ( + "testing" +) func Test_percentEncode(t *testing.T) { tests := []struct { diff --git a/xds/client/client.go b/xds/client/client.go index 1d9192378e..3d8e502f99 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -24,12 +24,18 @@ import ( "fmt" "sync" "time" +) +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" ) // clientImpl is the real implementation of the xds client. The exported Client @@ -40,8 +46,9 @@ import ( // style of ccBalancerWrapper so that the Client type does not implement these // exported methods. type clientImpl struct { - done *grpcsync.Event - config *bootstrap.Config + done *grpcsync.Event + config *bootstrap.Config + refreshMetadataCancel func() // authorityMu protects the authority fields. It's necessary because an // authority is created when it's used. @@ -91,6 +98,18 @@ func newWithConfig(config *bootstrap.Config, watchExpiryTimeout time.Duration, i return c, nil } +func (c *clientImpl) SetMetadata(m *_struct.Struct) error { + a, _, err := c.findAuthority(resource.ParseName("")) + if err != nil { + return err + } + if err := a.SetMetadata(m); err != nil { + return err + } + a.watchEndpoints("", func(update resource.EndpointsUpdate, err error) {}) + return nil +} + // BootstrapConfig returns the configuration read from the bootstrap file. // Callers must treat the return value as read-only. func (c *clientRefCounted) BootstrapConfig() *bootstrap.Config { diff --git a/xds/client/controller.go b/xds/client/controller.go index ad7b31cd4d..9f53a3b738 100644 --- a/xds/client/controller.go +++ b/xds/client/controller.go @@ -17,6 +17,10 @@ package client +import ( + _struct "github.com/golang/protobuf/ptypes/struct" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/controller" @@ -30,6 +34,7 @@ type controllerInterface interface { AddWatch(resourceType resource.ResourceType, resourceName string) RemoveWatch(resourceType resource.ResourceType, resourceName string) ReportLoad(server string) (*load.Store, func()) + SetMetadata(m *_struct.Struct) error Close() } diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go index da5362784f..78743722e3 100644 --- a/xds/client/controller/controller.go +++ b/xds/client/controller/controller.go @@ -29,7 +29,18 @@ import ( "fmt" "sync" "time" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + _struct "github.com/golang/protobuf/ptypes/struct" + + "google.golang.org/grpc" + "google.golang.org/grpc/keepalive" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/pubsub" @@ -37,8 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "google.golang.org/grpc" - "google.golang.org/grpc/keepalive" ) // Controller manages the connection and stream to the control plane. @@ -55,7 +64,7 @@ type Controller struct { logger *grpclog.PrefixLogger cc *grpc.ClientConn // Connection to the management server. - vClient version.VersionedClient + vClient version.MetadataWrappedVersionClient stopRunGoroutine context.CancelFunc backoff func(int) time.Duration @@ -154,6 +163,41 @@ func New(config *bootstrap.ServerConfig, updateHandler pubsub.UpdateHandler, val return ret, nil } +func (t *Controller) SetMetadata(m *_struct.Struct) error { + dopts := []grpc.DialOption{ + t.config.Creds, + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 5 * time.Minute, + Timeout: 20 * time.Second, + }), + } + + cc, err := grpc.Dial(t.config.ServerURI, dopts...) + if err != nil { + // An error from a non-blocking dial indicates something serious. + return fmt.Errorf("xds: failed to dial control plane {%s}: %v", t.config.ServerURI, err) + } + t.cc = cc + builder := version.GetAPIClientBuilder(t.config.TransportAPI) + if builder == nil { + return fmt.Errorf("no client builder for xDS API version: %v", t.config.TransportAPI) + } + v3PBNode, ok := t.config.NodeProto.(*v3corepb.Node) + if ok { + v3PBNode.Metadata = m + } + apiClient, err := builder(version.BuildOptions{NodeProto: t.config.NodeProto, Logger: t.logger}) + if err != nil { + return err + } + t.vClient = apiClient + t.stopRunGoroutine() + ctx, cancel := context.WithCancel(context.Background()) + t.stopRunGoroutine = cancel + go t.run(ctx) + return nil +} + // Close closes the controller. func (t *Controller) Close() { // Note that Close needs to check for nils even if some of them are always diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go index 5408cb123b..52d82fb320 100644 --- a/xds/client/controller/loadreport.go +++ b/xds/client/controller/loadreport.go @@ -19,10 +19,15 @@ package controller import ( "context" +) +import ( + "google.golang.org/grpc" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "google.golang.org/grpc" ) // ReportLoad starts an load reporting stream to the given server. If the server diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go index 0caed731e5..3953dfa20e 100644 --- a/xds/client/controller/transport.go +++ b/xds/client/controller/transport.go @@ -22,13 +22,19 @@ import ( "context" "fmt" "time" +) + +import ( + "github.com/golang/protobuf/proto" + + "google.golang.org/grpc" +) +import ( controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" ) // AddWatch adds a watch for an xDS resource given its type and name. diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go index df3fd610b7..d67e8b80ab 100644 --- a/xds/client/controller/version/v2/client.go +++ b/xds/client/controller/version/v2/client.go @@ -22,21 +22,30 @@ package v2 import ( "context" "fmt" +) + +import ( + v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" + v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" + v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" - controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + + statuspb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) - v2xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2" - v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" - v2adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" - statuspb "google.golang.org/genproto/googleapis/rpc/status" +import ( + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) func init() { @@ -52,7 +61,7 @@ var ( } ) -func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { +func newClient(opts controllerversion.BuildOptions) (controllerversion.MetadataWrappedVersionClient, error) { nodeProto, ok := opts.NodeProto.(*v2corepb.Node) if !ok { return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, (*v2corepb.Node)(nil)) @@ -71,6 +80,11 @@ type client struct { logger *grpclog.PrefixLogger } +// SetMetadata update client metadata +func (v2c *client) SetMetadata(p *_struct.Struct) { + v2c.nodeProto.Metadata = p +} + func (v2c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.ClientStream, error) { return v2adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) } diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go index 69d7b5df80..60cd6b3a05 100644 --- a/xds/client/controller/version/v2/loadreport.go +++ b/xds/client/controller/version/v2/loadreport.go @@ -23,20 +23,26 @@ import ( "errors" "fmt" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" v2endpointpb "github.com/envoyproxy/go-control-plane/envoy/api/v2/endpoint" lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v2" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go index 9fa804febc..ec15a7b711 100644 --- a/xds/client/controller/version/v3/client.go +++ b/xds/client/controller/version/v3/client.go @@ -22,21 +22,30 @@ package v3 import ( "context" "fmt" +) + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + statuspb "google.golang.org/genproto/googleapis/rpc/status" + "google.golang.org/grpc" "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" +) - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - v3adsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" - v3discoverypb "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" +import ( + controllerversion "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + resourceversion "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) func init() { @@ -52,7 +61,7 @@ var ( } ) -func newClient(opts controllerversion.BuildOptions) (controllerversion.VersionedClient, error) { +func newClient(opts controllerversion.BuildOptions) (controllerversion.MetadataWrappedVersionClient, error) { nodeProto, ok := opts.NodeProto.(*v3corepb.Node) if !ok { return nil, fmt.Errorf("xds: unsupported Node proto type: %T, want %T", opts.NodeProto, v3corepb.Node{}) @@ -77,6 +86,11 @@ func (v3c *client) NewStream(ctx context.Context, cc *grpc.ClientConn) (grpc.Cli return v3adsgrpc.NewAggregatedDiscoveryServiceClient(cc).StreamAggregatedResources(ctx, grpc.WaitForReady(true)) } +// SetMetadata update client metadata +func (v3c *client) SetMetadata(p *_struct.Struct) { + v3c.nodeProto.Metadata = p +} + // SendRequest sends out a DiscoveryRequest for the given resourceNames, of type // rType, on the provided stream. // diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go index 6a55f9ab46..800b10d2c1 100644 --- a/xds/client/controller/version/v3/loadreport.go +++ b/xds/client/controller/version/v3/loadreport.go @@ -23,20 +23,26 @@ import ( "errors" "fmt" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/load" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "github.com/golang/protobuf/proto" - "github.com/golang/protobuf/ptypes" - - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" lrsgrpc "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" lrspb "github.com/envoyproxy/go-control-plane/envoy/service/load_stats/v3" + + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/load" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + const clientFeatureLRSSendAllClusters = "envoy.lrs.supports_send_all_clusters" type lrsStream lrsgrpc.LoadReportingService_StreamLoadStatsClient diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go index 64f4f4dc19..439b9bb93c 100644 --- a/xds/client/controller/version/version.go +++ b/xds/client/controller/version/version.go @@ -21,18 +21,26 @@ package version import ( "context" "time" +) + +import ( + "github.com/golang/protobuf/proto" + _struct "github.com/golang/protobuf/ptypes/struct" + "google.golang.org/grpc" + + "google.golang.org/protobuf/types/known/anypb" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/load" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "github.com/golang/protobuf/proto" - "google.golang.org/grpc" - "google.golang.org/protobuf/types/known/anypb" ) var ( - m = make(map[version.TransportAPI]func(opts BuildOptions) (VersionedClient, error)) + m = make(map[version.TransportAPI]func(opts BuildOptions) (MetadataWrappedVersionClient, error)) ) // RegisterAPIClientBuilder registers a client builder for xDS transport protocol @@ -41,13 +49,13 @@ var ( // NOTE: this function must only be called during initialization time (i.e. in // an init() function), and is not thread-safe. If multiple builders are // registered for the same version, the one registered last will take effect. -func RegisterAPIClientBuilder(v version.TransportAPI, f func(opts BuildOptions) (VersionedClient, error)) { +func RegisterAPIClientBuilder(v version.TransportAPI, f func(opts BuildOptions) (MetadataWrappedVersionClient, error)) { m[v] = f } // GetAPIClientBuilder returns the client builder registered for the provided // xDS transport API version. -func GetAPIClientBuilder(version version.TransportAPI) func(opts BuildOptions) (VersionedClient, error) { +func GetAPIClientBuilder(version version.TransportAPI) func(opts BuildOptions) (MetadataWrappedVersionClient, error) { if f, ok := m[version]; ok { return f } @@ -121,3 +129,8 @@ type VersionedClient interface { // invoked. SendLoadStatsRequest(s grpc.ClientStream, loads []*load.Data) error } + +type MetadataWrappedVersionClient interface { + VersionedClient + SetMetadata(p *_struct.Struct) +} diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go index 46568591f9..9cb641a4c0 100644 --- a/xds/client/load/store_test.go +++ b/xds/client/load/store_test.go @@ -22,7 +22,9 @@ import ( "sort" "sync" "testing" +) +import ( "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" ) diff --git a/xds/client/logging.go b/xds/client/logging.go index 2b803f3c70..172d5858aa 100644 --- a/xds/client/logging.go +++ b/xds/client/logging.go @@ -20,11 +20,16 @@ package client import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-client %p] " var logger = grpclog.Component("xds") diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go index 7a8b82f425..490bde0709 100644 --- a/xds/client/pubsub/dump.go +++ b/xds/client/pubsub/dump.go @@ -18,10 +18,13 @@ package pubsub import ( - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" anypb "github.com/golang/protobuf/ptypes/any" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + func rawFromCache(s string, cache interface{}) *anypb.Any { switch c := cache.(type) { case map[string]resource.ListenerUpdate: diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go index 4226f581cf..5875a59525 100644 --- a/xds/client/pubsub/interface.go +++ b/xds/client/pubsub/interface.go @@ -17,7 +17,9 @@ package pubsub -import "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) // UpdateHandler receives and processes (by taking appropriate actions) xDS // resource updates from an APIClient for a specific version. diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go index d4412fd4ac..c14a1dcbbe 100644 --- a/xds/client/pubsub/pubsub.go +++ b/xds/client/pubsub/pubsub.go @@ -25,7 +25,9 @@ package pubsub import ( "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 42f8a7bd57..6765f65e9e 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -17,10 +17,13 @@ package pubsub +import ( + "google.golang.org/protobuf/proto" +) + import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" - "google.golang.org/protobuf/proto" ) type watcherInfoWithUpdate struct { @@ -59,6 +62,9 @@ func (pb *Pubsub) callCallback(wiu *watcherInfoWithUpdate) { ccb = func() { wiu.wi.rdsCallback(wiu.update.(resource.RouteConfigUpdate), wiu.err) } } case resource.ClusterResource: + if s, ok := pb.cdsWatchers["*"]; ok && s[wiu.wi] { + ccb = func() { wiu.wi.cdsCallback(wiu.update.(resource.ClusterUpdate), wiu.err) } + } if s, ok := pb.cdsWatchers[wiu.wi.target]; ok && s[wiu.wi] { ccb = func() { wiu.wi.cdsCallback(wiu.update.(resource.ClusterUpdate), wiu.err) } } @@ -186,7 +192,11 @@ func (pb *Pubsub) NewClusters(updates map[string]resource.ClusterUpdateErrTuple, defer pb.mu.Unlock() for name, uErr := range updates { - if s, ok := pb.cdsWatchers[name]; ok { + s, ok := pb.cdsWatchers[name] + if !ok { + s, ok = pb.cdsWatchers["*"] + } + if ok { if uErr.Err != nil { // On error, keep previous version for each resource. But update // status and error. diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go index 13cd8ed885..120e2231a5 100644 --- a/xds/client/pubsub/watch.go +++ b/xds/client/pubsub/watch.go @@ -21,7 +21,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" ) @@ -184,6 +186,10 @@ func (pb *Pubsub) watch(wi *watchInfo) (first bool, cancel func() bool) { wi.newUpdate(v) } case resource.ClusterResource: + if v, ok := pb.cdsCache["*"]; ok { + pb.logger.Debugf("CDS resource with name * found in cache: %+v", pretty.ToJSON(v)) + wi.newUpdate(v) + } if v, ok := pb.cdsCache[resourceName]; ok { pb.logger.Debugf("CDS resource with name %v found in cache: %+v", wi.target, pretty.ToJSON(v)) wi.newUpdate(v) diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go index be3269290e..23643bf1e4 100644 --- a/xds/client/resource/errors.go +++ b/xds/client/resource/errors.go @@ -18,7 +18,9 @@ package resource -import "fmt" +import ( + "fmt" +) // ErrorType is the type of the error that the watcher will receive from the xds // client. diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go index ccc786233e..c270aba0fa 100644 --- a/xds/client/resource/filter_chain.go +++ b/xds/client/resource/filter_chain.go @@ -18,22 +18,28 @@ package resource import ( - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" "errors" "fmt" "net" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +import ( v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +) + const ( // Used as the map key for unspecified prefixes. The actual value of this // key is immaterial. diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go index 9e720da1d5..56eff7b437 100644 --- a/xds/client/resource/locality_id.go +++ b/xds/client/resource/locality_id.go @@ -22,7 +22,9 @@ package resource import ( "encoding/json" "fmt" +) +import ( "google.golang.org/grpc/resolver" ) diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go index 163b3035b9..93ccda43f1 100644 --- a/xds/client/resource/matcher.go +++ b/xds/client/resource/matcher.go @@ -20,12 +20,17 @@ package resource import ( "fmt" "strings" +) +import ( + "google.golang.org/grpc/metadata" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" metedatautils "dubbo.apache.org/dubbo-go/v3/xds/utils/metadata" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" - "google.golang.org/grpc/metadata" ) // RouteToMatcher converts a route to a Matcher to match incoming RPC's against. diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go index 86d7253999..9310d3b7bf 100644 --- a/xds/client/resource/matcher_path.go +++ b/xds/client/resource/matcher_path.go @@ -20,7 +20,9 @@ package resource import ( "regexp" "strings" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" ) diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go index 629e8b2397..48a636e62a 100644 --- a/xds/client/resource/name.go +++ b/xds/client/resource/name.go @@ -21,7 +21,9 @@ import ( "net/url" "sort" "strings" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go index 262388af8f..7dff2dd430 100644 --- a/xds/client/resource/type.go +++ b/xds/client/resource/type.go @@ -19,11 +19,16 @@ package resource import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +) + // UpdateValidatorFunc performs validations on update structs using // context/logic available at the xdsClient layer. Since these validation are // performed on internal update structs, they can be shared between different diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go index a5fc9fc469..4d101817aa 100644 --- a/xds/client/resource/type_cds.go +++ b/xds/client/resource/type_cds.go @@ -17,7 +17,9 @@ package resource -import "google.golang.org/protobuf/types/known/anypb" +import ( + "google.golang.org/protobuf/types/known/anypb" +) // ClusterType is the type of cluster from a received CDS response. type ClusterType int diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go index fb5f3af56c..83e63505d5 100644 --- a/xds/client/resource/type_lds.go +++ b/xds/client/resource/type_lds.go @@ -19,11 +19,16 @@ package resource import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" +) + // ListenerUpdate contains information received in an LDS response, which is of // interest to the registered LDS watcher. type ListenerUpdate struct { diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go index 75efd46fcb..9b9b7ca806 100644 --- a/xds/client/resource/type_rds.go +++ b/xds/client/resource/type_rds.go @@ -20,12 +20,18 @@ package resource import ( "regexp" "time" +) + +import ( + "google.golang.org/grpc/codes" + + "google.golang.org/protobuf/types/known/anypb" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" - "google.golang.org/grpc/codes" - "google.golang.org/protobuf/types/known/anypb" ) // RouteConfigUpdate contains information received in an RDS response, which is diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go index c19f6e2ff1..29c09bbe3e 100644 --- a/xds/client/resource/unmarshal.go +++ b/xds/client/resource/unmarshal.go @@ -24,11 +24,16 @@ import ( "fmt" "strings" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + // UnmarshalOptions wraps the input parameters for `UnmarshalXxx` functions. type UnmarshalOptions struct { // Version is the version of the received response. diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go index e1dfc7934d..ef6e753532 100644 --- a/xds/client/resource/unmarshal_cds.go +++ b/xds/client/resource/unmarshal_cds.go @@ -22,20 +22,27 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3clusterpb "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3aggregateclusterpb "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" v3tlspb "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // TransportSocket proto message has a `name` field which is expected to be set // to this value by the management server. const transportSocketName = "envoy.transport_sockets.tls" @@ -81,6 +88,8 @@ const ( func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (ClusterUpdate, error) { var lbPolicy *ClusterLBPolicyRingHash + // todo @(laurence) this direct set + cluster.LbPolicy = v3clusterpb.Cluster_ROUND_ROBIN switch cluster.GetLbPolicy() { case v3clusterpb.Cluster_ROUND_ROBIN: lbPolicy = nil // The default is round_robin, and there's no config to set. @@ -134,6 +143,10 @@ func validateClusterAndConstructClusterUpdate(cluster *v3clusterpb.Cluster) (Clu } // Validate and set cluster type from the response. + // todo @laurence this set cluster + if x, ok := cluster.GetClusterDiscoveryType().(*v3clusterpb.Cluster_Type); ok { + x.Type = v3clusterpb.Cluster_EDS + } switch { case cluster.GetType() == v3clusterpb.Cluster_EDS: if cluster.GetEdsClusterConfig().GetEdsConfig().GetAds() == nil { diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go index f4403ca371..79a18f65a0 100644 --- a/xds/client/resource/unmarshal_eds.go +++ b/xds/client/resource/unmarshal_eds.go @@ -21,16 +21,23 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3endpointpb "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalEndpoints processes resources received in an EDS response, // validates them, and transforms them into a native struct which contains only // fields we are interested in. diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index 4e949e0f05..06d1d035a3 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -21,21 +21,30 @@ import ( "errors" "fmt" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v1udpatypepb "github.com/cncf/udpa/go/udpa/type/v1" + v3cncftypepb "github.com/cncf/xds/go/xds/type/v3" + v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3" v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalListener processes resources received in an LDS response, validates // them, and transforms them into a native struct which contains only fields we // are interested in. @@ -288,10 +297,10 @@ func processServerSideListener(lis *v3listenerpb.Listener, logger *grpclog.Prefi }, } - fcMgr, err := NewFilterChainManager(lis, logger) - if err != nil { - return nil, err - } - lu.InboundListenerCfg.FilterChains = fcMgr + //fcMgr, err := NewFilterChainManager(lis, logger) + //if err != nil { + // return nil, err + //} + //lu.InboundListenerCfg.FilterChains = fcMgr return lu, nil } diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go index e8063a6bee..9868bc46b4 100644 --- a/xds/client/resource/unmarshal_rds.go +++ b/xds/client/resource/unmarshal_rds.go @@ -22,19 +22,27 @@ import ( "regexp" "strings" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +import ( v3routepb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3typepb "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/protobuf/types/known/anypb" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" + "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" +) + // UnmarshalRouteConfig processes resources received in an RDS response, // validates them, and transforms them into a native struct which contains only // fields we are interested in. The provided hostname determines the route diff --git a/xds/client/singleton.go b/xds/client/singleton.go index 0a7ee3ca19..66daecfbec 100644 --- a/xds/client/singleton.go +++ b/xds/client/singleton.go @@ -24,7 +24,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" ) diff --git a/xds/csds/csds.go b/xds/csds/csds.go index 6bbb8e981f..90435de88a 100644 --- a/xds/csds/csds.go +++ b/xds/csds/csds.go @@ -26,22 +26,31 @@ package csds import ( "context" "io" +) - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +import ( v3adminpb "github.com/envoyproxy/go-control-plane/envoy/admin/v3" v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core" v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3statusgrpc "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" v3statuspb "github.com/envoyproxy/go-control-plane/envoy/service/status/v3" + "github.com/golang/protobuf/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/client" _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v2" // Register v2 xds_client. _ "dubbo.apache.org/dubbo-go/v3/xds/client/controller/version/v3" // Register v3 xds_client. + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) var ( diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go index f6f903824b..90352b4209 100644 --- a/xds/httpfilter/fault/fault.go +++ b/xds/httpfilter/fault/fault.go @@ -27,20 +27,29 @@ import ( "strconv" "sync/atomic" "time" +) + +import ( + cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" + fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" + tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" +) - cpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/common/fault/v3" - fpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/fault/v3" - tpb "github.com/envoyproxy/go-control-plane/envoy/type/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) const headerAbortHTTPStatus = "x-envoy-fault-abort-request" diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go index 4f167590cd..7302f83921 100644 --- a/xds/httpfilter/httpfilter.go +++ b/xds/httpfilter/httpfilter.go @@ -21,10 +21,13 @@ package httpfilter import ( - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" ) +import ( + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +) + // FilterConfig represents an opaque data structure holding configuration for a // filter. Embed this interface to implement it. type FilterConfig interface { diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go index eacf7361c6..a54e572103 100644 --- a/xds/httpfilter/rbac/rbac.go +++ b/xds/httpfilter/rbac/rbac.go @@ -24,17 +24,23 @@ import ( "errors" "fmt" "strings" +) + +import ( + v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - "dubbo.apache.org/dubbo-go/v3/xds/utils/rbac" - "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) - v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" - rpb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/rbac/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" + "dubbo.apache.org/dubbo-go/v3/xds/utils/rbac" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) func init() { diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go index 78a433a696..6a48af6dd2 100644 --- a/xds/httpfilter/router/router.go +++ b/xds/httpfilter/router/router.go @@ -21,14 +21,20 @@ package router import ( "fmt" +) + +import ( + pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" - "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "github.com/golang/protobuf/proto" "github.com/golang/protobuf/ptypes" + "google.golang.org/protobuf/types/known/anypb" +) - pb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/http/router/v3" +import ( + "dubbo.apache.org/dubbo-go/v3/xds/httpfilter" + iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" ) // TypeURL is the message type for the Router configuration. diff --git a/xds/internal.go b/xds/internal/internal.go similarity index 99% rename from xds/internal.go rename to xds/internal/internal.go index 776b948bb6..b778deb60f 100644 --- a/xds/internal.go +++ b/xds/internal/internal.go @@ -18,13 +18,16 @@ // Package internal contains gRPC-internal code, to avoid polluting // the godoc of the top-level grpc package. It must not import any grpc // symbols to avoid circular dependencies. -package xds +package internal import ( "context" "time" +) +import ( "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go index 1c3a947b0f..7a3b9bc081 100644 --- a/xds/resolver/logging.go +++ b/xds/resolver/logging.go @@ -20,11 +20,16 @@ package resolver import ( "fmt" +) - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/grpclog" ) +import ( + internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +) + const prefix = "[xds-resolver %p] " var logger = grpclog.Component("xds") diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go index 4640662c0d..5122bf444b 100644 --- a/xds/resolver/serviceconfig.go +++ b/xds/resolver/serviceconfig.go @@ -26,7 +26,19 @@ import ( "strings" "sync/atomic" "time" +) + +import ( + xxhash "github.com/cespare/xxhash/v2" + + "google.golang.org/grpc/codes" + + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/balancer/clustermanager" "dubbo.apache.org/dubbo-go/v3/xds/balancer/ringhash" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -37,10 +49,6 @@ import ( iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" "dubbo.apache.org/dubbo-go/v3/xds/utils/wrr" - xxhash "github.com/cespare/xxhash/v2" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" ) const ( diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go index 905aef7390..acb27b10d8 100644 --- a/xds/resolver/watch_service.go +++ b/xds/resolver/watch_service.go @@ -22,7 +22,9 @@ import ( "fmt" "sync" "time" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" "dubbo.apache.org/dubbo-go/v3/xds/clusterspecifier" diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go index 0627826dc2..f444d82e82 100644 --- a/xds/resolver/xds_resolver.go +++ b/xds/resolver/xds_resolver.go @@ -23,7 +23,15 @@ import ( "errors" "fmt" "strings" +) + +import ( + "google.golang.org/grpc/credentials" + + "google.golang.org/grpc/resolver" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" @@ -31,8 +39,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" "dubbo.apache.org/dubbo-go/v3/xds/utils/pretty" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/resolver" ) const xdsScheme = "xds" diff --git a/xds/server.go b/xds/server.go index caae913088..2977f764c9 100644 --- a/xds/server.go +++ b/xds/server.go @@ -24,10 +24,23 @@ import ( "fmt" "net" "sync" +) +import ( + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/grpclog" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/server" "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" @@ -35,13 +48,6 @@ import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" ) const serverPrefix = "[xds-server %p] " @@ -55,8 +61,8 @@ var ( return grpc.NewServer(opts...) } - grpcGetServerCreds = GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) - drainServerTransports = DrainServerTransports.(func(*grpc.Server, string)) + grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) + drainServerTransports = internal.DrainServerTransports.(func(*grpc.Server, string)) logger = grpclog.Component("xds") ) diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index 50ba42b19b..47fc8e7d58 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -24,10 +24,15 @@ import ( "net" "sync" "time" +) +import ( + "google.golang.org/grpc/credentials/tls/certprovider" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" xdsinternal "dubbo.apache.org/dubbo-go/v3/xds/utils/credentials/xds" - "google.golang.org/grpc/credentials/tls/certprovider" ) // connWrapper is a thin wrapper around a net.Conn returned by Accept(). It diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go index 971bc58cfe..3a7ded1fc1 100644 --- a/xds/server/listener_wrapper.go +++ b/xds/server/listener_wrapper.go @@ -28,16 +28,23 @@ import ( "sync/atomic" "time" "unsafe" +) + +import ( + "google.golang.org/grpc/backoff" + "google.golang.org/grpc/connectivity" + + "google.golang.org/grpc/grpclog" +) + +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" internalbackoff "dubbo.apache.org/dubbo-go/v3/xds/utils/backoff" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" - "google.golang.org/grpc/backoff" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/grpclog" ) var ( diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go index 552ac0e40d..96b891cfed 100644 --- a/xds/server/rds_handler.go +++ b/xds/server/rds_handler.go @@ -20,7 +20,9 @@ package server import ( "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) diff --git a/xds/server_options.go b/xds/server_options.go index 1d46c3adb7..8ebe349d98 100644 --- a/xds/server_options.go +++ b/xds/server_options.go @@ -20,7 +20,9 @@ package xds import ( "net" +) +import ( "google.golang.org/grpc" "google.golang.org/grpc/connectivity" ) diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go index 7cd305252a..9101eb9fe1 100644 --- a/xds/utils/backoff/backoff.go +++ b/xds/utils/backoff/backoff.go @@ -24,11 +24,16 @@ package backoff import ( "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" +import ( grpcbackoff "google.golang.org/grpc/backoff" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" +) + // Strategy defines the methodology for backing off after a grpc connection // failure. type Strategy interface { diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go index 950eaaa027..1a66fa9293 100644 --- a/xds/utils/balancer/stub/stub.go +++ b/xds/utils/balancer/stub/stub.go @@ -19,7 +19,9 @@ // Package stub implements a balancer for testing purposes. package stub -import "google.golang.org/grpc/balancer" +import ( + "google.golang.org/grpc/balancer" +) // BalancerFuncs contains all balancer.Balancer functions with a preceding // *BalancerData parameter for passing additional instance information. Any diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index 477ac7a0f4..1f27a12b52 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -22,14 +22,21 @@ import ( "fmt" "sync" "time" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/cache" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" + cache "dubbo.apache.org/dubbo-go/v3/xds/utils/xds_cache" +) + // subBalancerWrapper is used to keep the configurations that will be used to start // the underlying balancer. It can be called to start/stop the underlying // balancer. diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go index 9f6a0c1200..f056ab13ce 100644 --- a/xds/utils/buffer/unbounded.go +++ b/xds/utils/buffer/unbounded.go @@ -18,7 +18,9 @@ // Package buffer provides an implementation of an unbounded buffer. package buffer -import "sync" +import ( + "sync" +) // Unbounded is an implementation of an unbounded buffer which does not use // extra goroutines. This is typically used for passing updates from one entity diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index eaca00b818..f51cf05069 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -27,13 +27,20 @@ import ( "fmt" "strings" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +import ( "google.golang.org/grpc/attributes" + "google.golang.org/grpc/credentials/tls/certprovider" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + func init() { //internal.GetXDSHandshakeInfoForTesting = GetHandshakeInfo } diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go index 9e0683bb3e..55862c6968 100644 --- a/xds/utils/credentials/xds/handshake_info_test.go +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -24,7 +24,9 @@ import ( "net/url" "regexp" "testing" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" ) diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index 30a3b4258f..20eb1bcc8d 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -23,8 +23,17 @@ import ( "os" ) +import ( + "dubbo.apache.org/dubbo-go/v3/common/logger" +) + // Logger is the logger used for the non-depth log functions. -var Logger LoggerV2 +var Logger = func() logger.Logger { + logger.InitLogger(&logger.Config{ + CallerSkip: 2, + }) + return logger.GetLogger() +}() // DepthLogger is the logger used for the depth log functions. var DepthLogger DepthLoggerV2 @@ -34,7 +43,7 @@ func InfoDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.InfoDepth(depth, args...) } else { - Logger.Infoln(args...) + Logger.Info(args...) } } @@ -43,7 +52,7 @@ func WarningDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.WarningDepth(depth, args...) } else { - Logger.Warningln(args...) + Logger.Warn(args...) } } @@ -52,7 +61,7 @@ func ErrorDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.ErrorDepth(depth, args...) } else { - Logger.Errorln(args...) + Logger.Error(args...) } } @@ -61,7 +70,7 @@ func FatalDepth(depth int, args ...interface{}) { if DepthLogger != nil { DepthLogger.FatalDepth(depth, args...) } else { - Logger.Fatalln(args...) + Logger.Fatal(args...) } os.Exit(1) } diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go index 82af70e96f..0083dbf0e7 100644 --- a/xds/utils/grpclog/prefixLogger.go +++ b/xds/utils/grpclog/prefixLogger.go @@ -63,9 +63,6 @@ func (pl *PrefixLogger) Errorf(format string, args ...interface{}) { // Debugf does info logging at verbose level 2. func (pl *PrefixLogger) Debugf(format string, args ...interface{}) { - if !Logger.V(2) { - return - } if pl != nil { // Handle nil, so the tests can pass in a nil logger. format = pl.prefix + format diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go index 6f22bd8911..3d0ef900d8 100644 --- a/xds/utils/grpcutil/metadata.go +++ b/xds/utils/grpcutil/metadata.go @@ -20,7 +20,9 @@ package grpcutil import ( "context" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go index 7a092b2b80..410a4fe89e 100644 --- a/xds/utils/grpcutil/regex.go +++ b/xds/utils/grpcutil/regex.go @@ -18,7 +18,9 @@ package grpcutil -import "regexp" +import ( + "regexp" +) // FullMatchWithRegex returns whether the full text matches the regex provided. func FullMatchWithRegex(re *regexp.Regexp, text string) bool { diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go index 1043d5f81d..c17953f821 100644 --- a/xds/utils/hierarchy/hierarchy_test.go +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -20,9 +20,13 @@ package hierarchy import ( "testing" +) +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" ) diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go index 7d9bc0bd76..ead9a67bfd 100644 --- a/xds/utils/matcher/matcher_header.go +++ b/xds/utils/matcher/matcher_header.go @@ -20,12 +20,15 @@ package matcher import ( "fmt" - "google.golang.org/grpc/metadata" "regexp" "strconv" "strings" ) +import ( + "google.golang.org/grpc/metadata" +) + // HeaderMatcher is an interface for header matchers. These are // documented in (EnvoyProxy link here?). These matchers will match on different // aspects of HTTP header name/value pairs. diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go index f567f31982..f8de56fafb 100644 --- a/xds/utils/matcher/matcher_header_test.go +++ b/xds/utils/matcher/matcher_header_test.go @@ -21,7 +21,9 @@ package matcher import ( "regexp" "testing" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go index ad6d4b283b..52c022f731 100644 --- a/xds/utils/matcher/regex.go +++ b/xds/utils/matcher/regex.go @@ -18,7 +18,9 @@ package matcher -import "regexp" +import ( + "regexp" +) // FullMatchWithRegex returns whether the full text matches the regex provided. func FullMatchWithRegex(re *regexp.Regexp, text string) bool { diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go index 9703fd4cac..632064b8d9 100644 --- a/xds/utils/matcher/string_matcher.go +++ b/xds/utils/matcher/string_matcher.go @@ -25,11 +25,16 @@ import ( "fmt" "regexp" "strings" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcutil" +import ( v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcutil" +) + // StringMatcher contains match criteria for matching a string, and is an // internal representation of the `StringMatcher` proto defined at // https://github.com/envoyproxy/envoy/blob/main/api/envoy/type/matcher/v3/string.proto. diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go index 9528b57e44..d8d4dcfedd 100644 --- a/xds/utils/matcher/string_matcher_test.go +++ b/xds/utils/matcher/string_matcher_test.go @@ -21,8 +21,11 @@ package matcher import ( "regexp" "testing" +) +import ( v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/google/go-cmp/cmp" ) diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go index 9ebb1b22d5..232d92e588 100644 --- a/xds/utils/metadata/metadata.go +++ b/xds/utils/metadata/metadata.go @@ -20,7 +20,9 @@ package metadata import ( "context" +) +import ( "google.golang.org/grpc/metadata" ) diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go index 0177af4b51..e51a107388 100644 --- a/xds/utils/pretty/pretty.go +++ b/xds/utils/pretty/pretty.go @@ -23,10 +23,14 @@ import ( "bytes" "encoding/json" "fmt" +) +import ( "github.com/golang/protobuf/jsonpb" protov1 "github.com/golang/protobuf/proto" + "google.golang.org/protobuf/encoding/protojson" + protov2 "google.golang.org/protobuf/proto" ) diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go index edb39ea92b..7aed8f6877 100644 --- a/xds/utils/rbac/matchers.go +++ b/xds/utils/rbac/matchers.go @@ -21,14 +21,19 @@ import ( "fmt" "net" "regexp" +) - internalmatcher "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +import ( v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" v3route_componentspb "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" v3matcherpb "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" ) +import ( + internalmatcher "dubbo.apache.org/dubbo-go/v3/xds/utils/matcher" +) + // matcher is an interface that takes data about incoming RPC's and returns // whether it matches with whatever matcher implements this interface. type matcher interface { diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go index 3ed7895c03..ae3f73b1a4 100644 --- a/xds/utils/rbac/rbac_engine.go +++ b/xds/utils/rbac/rbac_engine.go @@ -27,9 +27,11 @@ import ( "fmt" "net" "strconv" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" +import ( v3rbacpb "github.com/envoyproxy/go-control-plane/envoy/config/rbac/v3" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -39,6 +41,10 @@ import ( "google.golang.org/grpc/status" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" +) + const logLevel = 2 var logger = grpclog.Component("rbac") diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go index 26417daad6..5ac81255a3 100644 --- a/xds/utils/resolver/config_selector.go +++ b/xds/utils/resolver/config_selector.go @@ -22,12 +22,18 @@ package resolver import ( "context" "sync" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +import ( "google.golang.org/grpc/metadata" + "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/serviceconfig" +) + // ConfigSelector controls what configuration to use for every RPC. type ConfigSelector interface { // Selects the configuration for the RPC, or terminates it using the error. diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go index 520d9229e1..56db8e9ff3 100644 --- a/xds/utils/resolver/passthrough/passthrough.go +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -20,7 +20,9 @@ // name without scheme back to gRPC as resolved address. package passthrough -import "google.golang.org/grpc/resolver" +import ( + "google.golang.org/grpc/resolver" +) const scheme = "passthrough" diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go index 10e2ee7222..d5e3af325a 100644 --- a/xds/utils/resolver/unix/unix.go +++ b/xds/utils/resolver/unix/unix.go @@ -21,11 +21,16 @@ package unix import ( "fmt" +) - "dubbo.apache.org/dubbo-go/v3/xds/utils/transport/networktype" +import ( "google.golang.org/grpc/resolver" ) +import ( + "dubbo.apache.org/dubbo-go/v3/xds/utils/transport/networktype" +) + const unixScheme = "unix" const unixAbstractScheme = "unix-abstract" diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go index badbdbf597..f9b6adcc67 100644 --- a/xds/utils/serviceconfig/serviceconfig.go +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -23,10 +23,15 @@ import ( "encoding/json" "fmt" "time" +) +import ( "google.golang.org/grpc/balancer" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/grpclog" + externalserviceconfig "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go index 3a725685db..6936bedfce 100644 --- a/xds/utils/serviceconfig/serviceconfig_test.go +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -22,9 +22,13 @@ import ( "encoding/json" "fmt" "testing" +) +import ( "github.com/google/go-cmp/cmp" + "google.golang.org/grpc/balancer" + externalserviceconfig "google.golang.org/grpc/serviceconfig" ) diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go index 61fbf4a5e7..459bf91040 100644 --- a/xds/utils/wrr/random.go +++ b/xds/utils/wrr/random.go @@ -20,7 +20,9 @@ package wrr import ( "fmt" "sync" +) +import ( "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcrand" ) diff --git a/xds/utils/xds_cache/timeoutCache.go b/xds/utils/xds_cache/timeoutCache.go new file mode 100644 index 0000000000..d1c0c293a0 --- /dev/null +++ b/xds/utils/xds_cache/timeoutCache.go @@ -0,0 +1,143 @@ +/* + * Copyright 2019 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds_cache + +import ( + "sync" + "time" +) + +type cacheEntry struct { + item interface{} + // Note that to avoid deadlocks (potentially caused by lock ordering), + // callback can only be called without holding cache's mutex. + callback func() + timer *time.Timer + // deleted is set to true in Remove() when the call to timer.Stop() fails. + // This can happen when the timer in the cache entry fires around the same + // time that timer.stop() is called in Remove(). + deleted bool +} + +// TimeoutCache is a cache with items to be deleted after a timeout. +type TimeoutCache struct { + mu sync.Mutex + timeout time.Duration + cache map[interface{}]*cacheEntry +} + +// NewTimeoutCache creates a TimeoutCache with the given timeout. +func NewTimeoutCache(timeout time.Duration) *TimeoutCache { + return &TimeoutCache{ + timeout: timeout, + cache: make(map[interface{}]*cacheEntry), + } +} + +// Add adds an item to the cache, with the specified callback to be called when +// the item is removed from the cache upon timeout. If the item is removed from +// the cache using a call to Remove before the timeout expires, the callback +// will not be called. +// +// If the Add was successful, it returns (newly added item, true). If there is +// an existing entry for the specified key, the cache entry is not be updated +// with the specified item and it returns (existing item, false). +func (c *TimeoutCache) Add(key, item interface{}, callback func()) (interface{}, bool) { + c.mu.Lock() + defer c.mu.Unlock() + if e, ok := c.cache[key]; ok { + return e.item, false + } + + entry := &cacheEntry{ + item: item, + callback: callback, + } + entry.timer = time.AfterFunc(c.timeout, func() { + c.mu.Lock() + if entry.deleted { + c.mu.Unlock() + // Abort the delete since this has been taken care of in Remove(). + return + } + delete(c.cache, key) + c.mu.Unlock() + entry.callback() + }) + c.cache[key] = entry + return item, true +} + +// Remove the item with the key from the cache. +// +// If the specified key exists in the cache, it returns (item associated with +// key, true) and the callback associated with the item is guaranteed to be not +// called. If the given key is not found in the cache, it returns (nil, false) +func (c *TimeoutCache) Remove(key interface{}) (item interface{}, ok bool) { + c.mu.Lock() + defer c.mu.Unlock() + entry, ok := c.removeInternal(key) + if !ok { + return nil, false + } + return entry.item, true +} + +// removeInternal removes and returns the item with key. +// +// caller must hold c.mu. +func (c *TimeoutCache) removeInternal(key interface{}) (*cacheEntry, bool) { + entry, ok := c.cache[key] + if !ok { + return nil, false + } + delete(c.cache, key) + if !entry.timer.Stop() { + // If stop was not successful, the timer has fired (this can only happen + // in a race). But the deleting function is blocked on c.mu because the + // mutex was held by the caller of this function. + // + // Set deleted to true to abort the deleting function. When the lock is + // released, the delete function will acquire the lock, check the value + // of deleted and return. + entry.deleted = true + } + return entry, true +} + +// Clear removes all entries, and runs the callbacks if runCallback is true. +func (c *TimeoutCache) Clear(runCallback bool) { + var entries []*cacheEntry + c.mu.Lock() + for key := range c.cache { + if e, ok := c.removeInternal(key); ok { + entries = append(entries, e) + } + } + c.mu.Unlock() + + if !runCallback { + return + } + + // removeInternal removes entries from cache, and also stops the timer, so + // the callback is guaranteed to be not called. If runCallback is true, + // manual execute all callbacks. + for _, entry := range entries { + entry.callback() + } +} diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index b28f376b50..0ba92b0ac7 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -18,6 +18,7 @@ package xds import ( "google.golang.org/grpc/attributes" + "google.golang.org/grpc/resolver" ) From 55374e75698af7a97fe2c8570a867e7792e01a1f Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Thu, 24 Mar 2022 20:42:30 +0800 Subject: [PATCH 09/19] fix: support router --- CHANGELOG.md | 4 +- cluster/router/meshrouter/factory.go | 41 +++ cluster/router/meshrouter/uniform_route.go | 185 +++++++++++++ common/constant/key.go | 7 +- common/url.go | 5 +- imports/imports.go | 3 +- protocol/invocation.go | 3 + protocol/invocation/rpcinvocation.go | 21 ++ registry/directory/directory.go | 35 ++- registry/xds/registry.go | 8 +- remoting/xds/client.go | 293 +++++++++++++++++---- remoting/xds/ewatcher.go | 41 +++ test/xds/main.go | 8 +- xds/client/pubsub/update.go | 17 ++ xds/utils/envconfig/xds.go | 2 +- 15 files changed, 605 insertions(+), 68 deletions(-) create mode 100644 cluster/router/meshrouter/factory.go create mode 100644 cluster/router/meshrouter/uniform_route.go create mode 100644 remoting/xds/ewatcher.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fd6ce2d8a3..e40a5cdf6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -345,7 +345,7 @@ Milestone: [https://github.com/apache/dubbo-go/milestone/5](https://github.com/a - [Fix go client quit abnormally when it connects java server](https://github.com/apache/dubbo-go/pull/820) [@wenxuwan](https://github.com/wenxuwan) - [Fix sentinel windows issue](https://github.com/apache/dubbo-go/pull/821) [@louyuting](https://github.com/louyuting) - [Fix metadata default port](https://github.com/apache/dubbo-go/pull/821) [@sanxun0325](https://github.com/sanxun0325) -- [Fix consul can not destory](https://github.com/apache/dubbo-go/pull/788) [@LaurenceLiZhixin](https://github.com/LaurenceLiZhixin) +- [Fix consul can not destroy](https://github.com/apache/dubbo-go/pull/788) [@LaurenceLiZhixin](https://github.com/LaurenceLiZhixin) Milestone: [https://github.com/apache/dubbo-go/milestone/6](https://github.com/apache/dubbo-go/milestone/6?closed=1) @@ -390,7 +390,7 @@ Project: [https://github.com/apache/dubbo-go/projects/10](https://github.com/apa - [Fix go client quit abnormally when it connects java server](https://github.com/apache/dubbo-go/pull/820) [@wenxuwan](https://github.com/wenxuwan) - [Fix sentinel windows issue](https://github.com/apache/dubbo-go/pull/821) [@louyuting](https://github.com/louyuting) - [Fix metadata default port](https://github.com/apache/dubbo-go/pull/821) [@sanxun0325](https://github.com/sanxun0325) -- [Fix consul can not destory](https://github.com/apache/dubbo-go/pull/788) [@LaurenceLiZhixin](https://github.com/LaurenceLiZhixin) +- [Fix consul can not destroy](https://github.com/apache/dubbo-go/pull/788) [@LaurenceLiZhixin](https://github.com/LaurenceLiZhixin) Milestone: [https://github.com/apache/dubbo-go/milestone/6](https://github.com/apache/dubbo-go/milestone/6?closed=1) diff --git a/cluster/router/meshrouter/factory.go b/cluster/router/meshrouter/factory.go new file mode 100644 index 0000000000..e5c8961a74 --- /dev/null +++ b/cluster/router/meshrouter/factory.go @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package meshrouter + +import ( + "dubbo.apache.org/dubbo-go/v3/cluster/router" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/extension" +) + +func init() { + extension.SetRouterFactory(constant.MeshRouterFactoryKey, NewMeshRouterFactory) +} + +// UniformRouteFactory is uniform router's factory +type MeshRouterFactory struct{} + +// NewMeshRouterFactory constructs a new PriorityRouterFactory +func NewMeshRouterFactory() router.PriorityRouterFactory { + return &MeshRouterFactory{} +} + +// NewPriorityRouter construct a new UniformRouteFactory as PriorityRouter +func (f *MeshRouterFactory) NewPriorityRouter() (router.PriorityRouter, error) { + return NewMeshRouter() +} diff --git a/cluster/router/meshrouter/uniform_route.go b/cluster/router/meshrouter/uniform_route.go new file mode 100644 index 0000000000..0c635fe324 --- /dev/null +++ b/cluster/router/meshrouter/uniform_route.go @@ -0,0 +1,185 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package meshrouter + +import ( + "bytes" + "math/rand" + "strings" +) + +import ( + perrors "github.com/pkg/errors" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/cluster/router" + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/config_center" + "dubbo.apache.org/dubbo-go/v3/protocol" + "dubbo.apache.org/dubbo-go/v3/remoting/xds" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" + "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" +) + +const ( + name = "mesh-router" +) + +// MeshRouter have +type MeshRouter struct { + client *xds.WrappedClient +} + +// NewMeshRouter construct an NewConnCheckRouter via url +func NewMeshRouter() (router.PriorityRouter, error) { + xdsWrappedClient := xds.GetXDSWrappedClient() + if xdsWrappedClient == nil { + return nil, perrors.Errorf("[Mesh Router] xds wrapped client is not created.") + } + return &MeshRouter{ + client: xdsWrappedClient, + }, nil +} + +// Route gets a list of routed invoker +func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocation protocol.Invocation) []protocol.Invoker { + hostAddr, err := r.client.GetHostAddrByServiceUniqueKey(getSubscribeName(url)) + if err != nil { + // todo deal with error + return nil + } + rconf := r.client.GetRouterConfig(hostAddr) + + clusterInvokerMap := make(map[string][]protocol.Invoker) + for _, v := range invokers { + meshClusterId := v.GetURL().GetParam("meshClusterId", "") + if _, ok := clusterInvokerMap[meshClusterId]; !ok { + clusterInvokerMap[meshClusterId] = make([]protocol.Invoker, 0) + } + clusterInvokerMap[meshClusterId] = append(clusterInvokerMap[meshClusterId], v) + } + + if len(rconf.VirtualHosts) != 0 { + // try to route to sub virtual host + for _, vh := range rconf.VirtualHosts { + // 1. match domain + //vh.Domains == ["*"] + + // 2. match http route + for _, r := range vh.Routes { + //route. + ctx := invocation.ToContext() + matcher, err := resource.RouteToMatcher(r) + if err != nil { + panic(err) + } + if matcher.Match(resolver.RPCInfo{ + Context: ctx, + Method: "/" + invocation.MethodName(), + }) { + // Loop through routes in order and select first match. + if r == nil || r.WeightedClusters == nil { + panic("cluster notfound") + } + invokersWeightPairs := make(invokerWeightPairs, 0) + + for clusterId, weight := range r.WeightedClusters { + // cluster -> invokers + targetInvokers := clusterInvokerMap[clusterId] + invokersWeightPairs = append(invokersWeightPairs, invokerWeightPair{ + invokers: targetInvokers, + weight: weight.Weight, + }) + } + return invokersWeightPairs.GetInvokers() + } + } + } + } + return invokers +} + +// Process there is no process needs for uniform Router, as it upper struct RouterChain has done it +func (r *MeshRouter) Process(event *config_center.ConfigChangeEvent) { +} + +// Name get name of ConnCheckerRouter +func (r *MeshRouter) Name() string { + return name +} + +// Priority get Router priority level +func (r *MeshRouter) Priority() int64 { + return 0 +} + +// URL Return URL in router +func (r *MeshRouter) URL() *common.URL { + return nil +} + +// Notify the router the invoker list +func (r *MeshRouter) Notify(invokers []protocol.Invoker) { +} + +type invokerWeightPair struct { + invokers []protocol.Invoker + weight uint32 +} + +type invokerWeightPairs []invokerWeightPair + +func (i *invokerWeightPairs) GetInvokers() []protocol.Invoker { + if len(*i) == 0 { + return nil + } + totalWeight := uint32(0) + tempWeight := uint32(0) + for _, v := range *i { + totalWeight += v.weight + } + randFloat := rand.Float64() + for _, v := range *i { + tempWeight += v.weight + tempPercent := float64(tempWeight) / float64(totalWeight) + if tempPercent >= randFloat { + return v.invokers + } + } + return (*i)[0].invokers +} + +func getSubscribeName(url *common.URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(common.DubboNodes[common.PROVIDER])) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + +func appendParam(target *bytes.Buffer, url *common.URL, key string) { + value := url.GetParam(key, "") + target.Write([]byte(constant.NacosServiceNameSeparator)) + if strings.TrimSpace(value) != "" { + target.Write([]byte(value)) + } +} diff --git a/common/constant/key.go b/common/constant/key.go index 0e1a684d3f..1a1c8be520 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -305,9 +305,10 @@ const ( ForceUseTag = "dubbo.force.tag" Tagkey = "dubbo.tag" // AttachmentKey in context in invoker - AttachmentKey = DubboCtxKey("attachment") - TagRouterFactoryKey = "tag" - V3RouterFactoryKey = "mesh" + AttachmentKey = DubboCtxKey("attachment") + TagRouterFactoryKey = "tag" + V3RouterFactoryKey = "v3router" + MeshRouterFactoryKey = "mesh" ) // Auth filter diff --git a/common/url.go b/common/url.go index a800a3093d..7d63c6fd62 100644 --- a/common/url.go +++ b/common/url.go @@ -367,9 +367,10 @@ func (c *URL) Key() string { func (c *URL) GetCacheInvokerMapKey() string { urlNew, _ := NewURL(c.PrimitiveURL) - buildString := fmt.Sprintf("%s://%s:%s@%s:%s/?interface=%s&group=%s&version=%s×tamp=%s", + buildString := fmt.Sprintf("%s://%s:%s@%s:%s/?interface=%s&group=%s&version=%s×tamp=%s&meshClusterId=%s", c.Protocol, c.Username, c.Password, c.Ip, c.Port, c.Service(), c.GetParam(constant.GroupKey, ""), - c.GetParam(constant.VersionKey, ""), urlNew.GetParam(constant.TimestampKey, "")) + c.GetParam(constant.VersionKey, ""), urlNew.GetParam(constant.TimestampKey, ""), + c.GetParam("meshClusterId", "")) return buildString } diff --git a/imports/imports.go b/imports/imports.go index ab40b61968..12cbd31c40 100644 --- a/imports/imports.go +++ b/imports/imports.go @@ -32,8 +32,7 @@ import ( _ "dubbo.apache.org/dubbo-go/v3/cluster/loadbalance/p2c" _ "dubbo.apache.org/dubbo-go/v3/cluster/loadbalance/random" _ "dubbo.apache.org/dubbo-go/v3/cluster/loadbalance/roundrobin" - _ "dubbo.apache.org/dubbo-go/v3/cluster/router/tag" - _ "dubbo.apache.org/dubbo-go/v3/cluster/router/v3router" + _ "dubbo.apache.org/dubbo-go/v3/cluster/router/meshrouter" _ "dubbo.apache.org/dubbo-go/v3/common/proxy/proxy_factory" _ "dubbo.apache.org/dubbo-go/v3/config_center/apollo" _ "dubbo.apache.org/dubbo-go/v3/config_center/nacos" diff --git a/protocol/invocation.go b/protocol/invocation.go index 4580250d32..ddbaa29049 100644 --- a/protocol/invocation.go +++ b/protocol/invocation.go @@ -18,6 +18,7 @@ package protocol import ( + "context" "reflect" ) @@ -57,4 +58,6 @@ type Invocation interface { SetAttribute(key string, value interface{}) GetAttribute(key string) (interface{}, bool) GetAttributeWithDefaultValue(key string, defaultValue interface{}) interface{} + + ToContext() context.Context } diff --git a/protocol/invocation/rpcinvocation.go b/protocol/invocation/rpcinvocation.go index 399e3a1335..6af2ab367d 100644 --- a/protocol/invocation/rpcinvocation.go +++ b/protocol/invocation/rpcinvocation.go @@ -18,11 +18,16 @@ package invocation import ( + "context" "reflect" "strings" "sync" ) +import ( + "google.golang.org/grpc/metadata" +) + import ( "dubbo.apache.org/dubbo-go/v3/common" "dubbo.apache.org/dubbo-go/v3/common/constant" @@ -239,6 +244,22 @@ func (r *RPCInvocation) GetAttributeWithDefaultValue(key string, defaultValue in return defaultValue } +func (r *RPCInvocation) ToContext() context.Context { + gRPCMD := make(metadata.MD, 0) + ctx := context.Background() + for k, v := range r.Attachments() { + if str, ok := v.(string); ok { + gRPCMD.Set(k, str) + continue + } + if str, ok := v.([]string); ok { + gRPCMD.Set(k, str...) + continue + } + } + return metadata.NewOutgoingContext(ctx, gRPCMD) +} + // ///////////////////////// // option // ///////////////////////// diff --git a/registry/directory/directory.go b/registry/directory/directory.go index 56509ec74a..ebb2078710 100644 --- a/registry/directory/directory.go +++ b/registry/directory/directory.go @@ -129,13 +129,15 @@ func (dir *RegistryDirectory) refreshInvokers(event *registry.ServiceEvent) { logger.Debug("refresh invokers with nil") } - var oldInvoker protocol.Invoker + var oldInvoker []protocol.Invoker if event != nil { oldInvoker, _ = dir.cacheInvokerByEvent(event) } dir.setNewInvokers() - if oldInvoker != nil { - oldInvoker.Destroy() + for _, v := range oldInvoker { + if v != nil { + v.Destroy() + } } } @@ -238,7 +240,7 @@ func (dir *RegistryDirectory) setNewInvokers() { } // cacheInvokerByEvent caches invokers from the service event -func (dir *RegistryDirectory) cacheInvokerByEvent(event *registry.ServiceEvent) (protocol.Invoker, error) { +func (dir *RegistryDirectory) cacheInvokerByEvent(event *registry.ServiceEvent) ([]protocol.Invoker, error) { // judge is override or others if event != nil { @@ -249,7 +251,7 @@ func (dir *RegistryDirectory) cacheInvokerByEvent(event *registry.ServiceEvent) if u != nil && constant.RouterProtocol == u.Protocol { dir.configRouters() } - return dir.cacheInvoker(u, event), nil + return []protocol.Invoker{dir.cacheInvoker(u, event)}, nil case remoting.EventTypeDel: logger.Infof("[Registry Directory] selector delete service url{%s}", event.Service) return dir.uncacheInvoker(event), nil @@ -314,9 +316,28 @@ func (dir *RegistryDirectory) toGroupInvokers() []protocol.Invoker { return groupInvokersList } +func (dir *RegistryDirectory) uncacheInvokerWithClusterId(clusterId string) []protocol.Invoker { + logger.Debugf("All service will be deleted in cache invokers with clusterId %s!", clusterId) + invokerKeys := make([]string, 0) + dir.cacheInvokersMap.Range(func(key, cacheInvoker interface{}) bool { + if cacheInvoker.(protocol.Invoker).GetURL().GetParam("meshClusterId", "") == clusterId { + invokerKeys = append(invokerKeys, key.(string)) + } + return true + }) + uncachedInvokers := make([]protocol.Invoker, 0) + for _, v := range invokerKeys { + uncachedInvokers = append(uncachedInvokers, dir.uncacheInvokerWithKey(v)) + } + return uncachedInvokers +} + // uncacheInvoker will return abandoned Invoker, if no Invoker to be abandoned, return nil -func (dir *RegistryDirectory) uncacheInvoker(event *registry.ServiceEvent) protocol.Invoker { - return dir.uncacheInvokerWithKey(event.Key()) +func (dir *RegistryDirectory) uncacheInvoker(event *registry.ServiceEvent) []protocol.Invoker { + if clusterId := event.Service.GetParam("meshClusterId", ""); event.Service.Location == "*" && clusterId != "" { + dir.uncacheInvokerWithClusterId(clusterId) + } + return []protocol.Invoker{dir.uncacheInvokerWithKey(event.Key())} } func (dir *RegistryDirectory) uncacheInvokerWithKey(key string) protocol.Invoker { diff --git a/registry/xds/registry.go b/registry/xds/registry.go index b0050da665..f4b5ba4730 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -134,7 +134,7 @@ func (nr *xdsRegistry) GetURL() *common.URL { func (nr *xdsRegistry) getHostAddrFromURL(url *common.URL) (string, string, error) { svcName := getSubscribeName(url) - hostAddr, err := nr.xdsWrappedClient.GetHostAddrFromPilot(svcName) + hostAddr, err := nr.xdsWrappedClient.GetHostAddrByServiceUniqueKey(svcName) return hostAddr, svcName, err } @@ -170,9 +170,11 @@ func newXDSRegistry(url *common.URL) (registry.Registry, error) { if err != nil { return nil, err } - tmpRegistry := &xdsRegistry{ + + newRegistry := &xdsRegistry{ xdsWrappedClient: wrappedXDSClient, registryURL: url, } - return tmpRegistry, nil + + return newRegistry, nil } diff --git a/remoting/xds/client.go b/remoting/xds/client.go index b8435cab61..4c5c9a4582 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -38,6 +38,8 @@ const ( clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) +var xdsWrappedClient *WrappedClient + type WrappedClient struct { interfaceAppNameMap map[string]string subscribeStopChMap sync.Map @@ -51,9 +53,35 @@ type WrappedClient struct { lock sync.Mutex endpointClusterMap sync.Map + + edsMap map[string]resource.EndpointsUpdate // edsMap is used to send delete signal if one cluster is deleted + edsRWLock sync.RWMutex + rdsMap map[string]resource.RouteConfigUpdate // rdsMap is used by mesh router + rdsRWLock sync.RWMutex + cdsMap map[string]resource.ClusterUpdate // cdsMap is full clusterId -> clusterUpdate cache of this istio system + cdsRWLock sync.RWMutex + + cdsUpdateEventChan chan struct{} + cdsUpdateEventHandlers []func() + + hostAddrListenerMap map[string]map[string]registry.NotifyListener + hostAddrListenerMapLock sync.RWMutex + hostAddrClusterCtxMap map[string]map[string]endPointWatcherCtx + hostAddrClusterCtxMapLock sync.RWMutex + + interfaceNameHostAddrMap map[string]string + interfaceNameHostAddrMapLock sync.RWMutex +} + +func GetXDSWrappedClient() *WrappedClient { + return xdsWrappedClient } func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*WrappedClient, error) { + // todo @(laurence) safety problem? what if to concurrent 'new' both create new client? + if xdsWrappedClient != nil { + return xdsWrappedClient, nil + } // get hostname from http://localhost:8080/debug/endpointz newClient := &WrappedClient{ podName: podName, @@ -61,14 +89,29 @@ func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*W localIP: localIP, istiodHostName: istiodHostName, interfaceAppNameMap: make(map[string]string), + + rdsMap: make(map[string]resource.RouteConfigUpdate), + cdsMap: make(map[string]resource.ClusterUpdate), + edsMap: make(map[string]resource.EndpointsUpdate), + + hostAddrListenerMap: make(map[string]map[string]registry.NotifyListener), + hostAddrClusterCtxMap: make(map[string]map[string]endPointWatcherCtx), + + interfaceNameHostAddrMap: make(map[string]string), + cdsUpdateEventChan: make(chan struct{}), + cdsUpdateEventHandlers: make([]func(), 0), } - if err := newClient.initClientAndloadLocalHostAddr(); err != nil { + // todo gr control + newClient.startWatchingResource() + if err := newClient.initClientAndLoadLocalHostAddr(); err != nil { return nil, err } + + xdsWrappedClient = newClient return newClient, nil } -func (w *WrappedClient) getInterfaceHostAddrMapFromPilot() (map[string]string, error) { +func (w *WrappedClient) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { req, _ := http.NewRequest(http.MethodGet, "http://"+w.istiodPodIP+":8080/debug/adsz", nil) token, err := os.ReadFile("/var/run/secrets/token/istio-token") if err != nil { @@ -82,6 +125,9 @@ func (w *WrappedClient) getInterfaceHostAddrMapFromPilot() (map[string]string, e } data, err := ioutil.ReadAll(rsp.Body) + if err != nil{ + return nil, err + } adszRsp := &ADSZResponse{} if err := json.Unmarshal(data, adszRsp); err != nil { return nil, err @@ -89,15 +135,19 @@ func (w *WrappedClient) getInterfaceHostAddrMapFromPilot() (map[string]string, e return adszRsp.GetMap(), nil } -// GetHostAddrFromPilot todo 1. timeout 2. hostAddr change? -func (w *WrappedClient) GetHostAddrFromPilot(serviceKey string) (string, error) { +// GetHostAddrByServiceUniqueKey todo 1. timeout 2. hostAddr change? +func (w *WrappedClient) GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) { + if hostAddr, ok := w.interfaceNameHostAddrMap[serviceUniqueKey]; ok { + return hostAddr, nil + } for { - if interfaceHostMap, err := w.getInterfaceHostAddrMapFromPilot(); err != nil { + if interfaceHostAddrMap, err := w.getServiceUniqueKeyHostAddrMapFromPilot(); err != nil { return "", err } else { - hostName, ok := interfaceHostMap[serviceKey] + w.interfaceNameHostAddrMap = interfaceHostAddrMap + hostName, ok := interfaceHostAddrMap[serviceUniqueKey] if !ok { - logger.Infof("[XDS Wrapped Client] Try getting service %s 's host from istio %d\n", serviceKey, w.istiodHostName) + logger.Infof("[XDS Wrapped Client] Try getting interface %s 's host from istio %d\n", serviceUniqueKey, w.istiodHostName) time.Sleep(time.Millisecond * 100) continue } @@ -106,40 +156,163 @@ func (w *WrappedClient) GetHostAddrFromPilot(serviceKey string) (string, error) } } -func (w *WrappedClient) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { - _, ok := w.subscribeStopChMap.Load(svcUniqueName) - if ok { - return perrors.Errorf("XDS WrappedClient subscribe interface %s failed, subscription already exist.", interfaceName) - } - stopCh := make(chan struct{}) - w.subscribeStopChMap.Store(svcUniqueName, stopCh) +func getHostNameAndPortFromAddr(hostAddr string) (string, string) { ipPort := strings.Split(hostAddr, ":") hostName := ipPort[0] port := ipPort[1] - // todo cds, eds - cancel := w.xdsClient.WatchEndpoints(fmt.Sprintf("outbound|%s||%s", port, hostName), func(update resource.EndpointsUpdate, err error) { - for _, v := range update.Localities { - for _, e := range v.Endpoints { - // todo 1. register protocol in server side metadata 2. get metadata from endpoint, for router - url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", e.Address, interfaceName)) - logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", - interfaceName, e.Address, e.HealthStatus) - if e.HealthStatus == resource.EndpointHealthStatusHealthy { - lst.Notify(®istry.ServiceEvent{ - Action: remoting.EventTypeUpdate, - Service: url, - }) - } else { - lst.Notify(®istry.ServiceEvent{ - Action: remoting.EventTypeDel, - Service: url, - }) + return hostName, port +} + +func (w *WrappedClient) getAllVersionClusterName(hostAddr string) []string { + hostName, port := getHostNameAndPortFromAddr(hostAddr) + allVersionClusterNames := make([]string, 0) + for clusterName, _ := range w.cdsMap { + clusterNameData := strings.Split(clusterName, "|") + if clusterNameData[1] == port && clusterNameData[3] == hostName { + allVersionClusterNames = append(allVersionClusterNames, clusterName) + } + } + return allVersionClusterNames +} + +func generateRegistryEvent(clusterName string, endpoint resource.Endpoint, interfaceName string) *registry.ServiceEvent { + // todo 1. register protocol in server side metadata 2. get metadata from endpoint，like label, for router + url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", endpoint.Address, interfaceName)) + logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", + interfaceName, endpoint.Address, endpoint.HealthStatus) + clusterNames := strings.Split(clusterName, "|") + // todo const MeshSubsetKey + url.AddParam("meshSubset", clusterNames[2]) + url.AddParam("meshClusterId", clusterName) + url.AddParam("meshHostAddr", clusterNames[3]+":"+clusterNames[1]) + if endpoint.HealthStatus == resource.EndpointHealthStatusUnhealthy { + return ®istry.ServiceEvent{ + Action: remoting.EventTypeDel, + Service: url, + } + } + return ®istry.ServiceEvent{ + Action: remoting.EventTypeUpdate, + Service: url, + } +} + +// registerHostLevelSubscription register: 1. all related cluster, 2. router config +func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName string, lst registry.NotifyListener) { + // 1. listen all cluster related endpoint + if _, ok := w.hostAddrListenerMap[hostAddr]; ok { + // if subscription exist, register listener directly + w.hostAddrListenerMap[hostAddr][svcUniqueName] = lst + return + } + // host Addr key must not exist in map, create one + w.hostAddrListenerMap[hostAddr] = make(map[string]registry.NotifyListener) + w.hostAddrClusterCtxMap[hostAddr] = make(map[string]endPointWatcherCtx) + w.hostAddrListenerMap[hostAddr][svcUniqueName] = lst + + // watch cluster change, and start listening newcoming cluster + w.cdsUpdateEventHandlers = append(w.cdsUpdateEventHandlers, func() { + // todo @(laurnece) now this event would be called if any cluster is change, but not only this hostAddr's + updatedAllVersionedClusterName := w.getAllVersionClusterName(hostAddr) + // do patch + listeningClustersCancelMap := w.hostAddrClusterCtxMap[hostAddr] + oldlisteningClusterMap := make(map[string]bool) + for cluster, _ := range listeningClustersCancelMap { + oldlisteningClusterMap[cluster] = false + } + for _, updatedClusterName := range updatedAllVersionedClusterName { + if _, ok := listeningClustersCancelMap[updatedClusterName]; ok { + // already listening + oldlisteningClusterMap[updatedClusterName] = true + continue + } + // new cluster + watcher := endPointWatcherCtx{ + interfaceName: interfaceName, + clusterName: updatedClusterName, + hostAddr: hostAddr, + hostAddrListenerMap: w.hostAddrListenerMap, + xdsClient: w, + } + cancel := w.xdsClient.WatchEndpoints(updatedClusterName, watcher.handle) + watcher.cancel = cancel + w.hostAddrClusterCtxMap[hostAddr][updatedClusterName] = watcher + } + + // cancel not exist cluster + for cluster, v := range oldlisteningClusterMap { + if !v { + // this cluster not exist in update cluster list + if watchCtx, ok := w.hostAddrClusterCtxMap[hostAddr][cluster]; ok { + delete(w.hostAddrClusterCtxMap[hostAddr], cluster) + watchCtx.destroy() } } } }) + + // update cluster of now + allVersionedClusterName := w.getAllVersionClusterName(hostAddr) + for _, c := range allVersionedClusterName { + watcher := endPointWatcherCtx{ + interfaceName: interfaceName, + clusterName: c, + hostAddr: hostAddr, + hostAddrListenerMap: w.hostAddrListenerMap, + } + watcher.cancel = w.xdsClient.WatchEndpoints(c, watcher.handle) + + w.hostAddrClusterCtxMap[hostAddr][c] = watcher + } + + // 2. cache route config + // todo @(laurnece) cancel watching of this addr's rds + _ = w.xdsClient.WatchRouteConfig(hostAddr, func(update resource.RouteConfigUpdate, err error) { + if update.VirtualHosts == nil { + return + } + w.rdsMap[hostAddr] = update + }) +} + +func (w *WrappedClient) GetRouterConfig(hostAddr string) resource.RouteConfigUpdate { + rconf, ok := w.rdsMap[hostAddr] + if ok { + return rconf + } + return resource.RouteConfigUpdate{} +} + +func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName string) { + if _, ok := w.hostAddrListenerMap[hostAddr]; ok { + // if subscription exist, register listener directly + if _, exist := w.hostAddrListenerMap[hostAddr][svcUniqueName]; exist { + delete(w.hostAddrListenerMap[hostAddr], svcUniqueName) + } + if (len(w.hostAddrListenerMap[hostAddr])) == 0 { + // if no subscription of this host cancel all cds subscription of this hostAddr + keys := make([]string, 0) + for k, c := range w.hostAddrClusterCtxMap[hostAddr] { + c.destroy() + keys = append(keys, k) + } + for _, v := range keys { + delete(w.hostAddrClusterCtxMap, v) + } + } + } +} + +func (w *WrappedClient) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { + _, ok := w.subscribeStopChMap.Load(svcUniqueName) + if ok { + return perrors.Errorf("XDS WrappedClient subscribe interface %s failed, subscription already exist.", interfaceName) + } + stopCh := make(chan struct{}) + w.subscribeStopChMap.Store(svcUniqueName, stopCh) + w.registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName, lst) <-stopCh - cancel() + w.unregisterHostLevelSubscription(hostAddr, svcUniqueName) return nil } @@ -179,25 +352,41 @@ func (w *WrappedClient) ChangeInterfaceMap(interfaceName string, add bool) error return nil } -func (w *WrappedClient) initClientAndloadLocalHostAddr() error { +func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { // call watch and refresh istiod debug interface xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) if err != nil { return err } - stopCh := make(chan struct{}) + foundLocalStopCh := make(chan struct{}) + foundIstiodStopCh := make(chan struct{}) foundLocal := false foundIstiod := false - cancel := xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + var cancel1 func() + var cancel2 func() + // todo @(laurence) here, if istiod is unhealthy, here should be timeout and tell developer. + _ = xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { if update.ClusterName == "" { return } + if update.ClusterName[:1] == "-" { + // delete event + delete(w.cdsMap, update.ClusterName[1:]) + w.cdsUpdateEventChan <- struct{}{} // send update event + return + } + w.cdsMap[update.ClusterName] = update + w.cdsUpdateEventChan <- struct{}{} // send update event + if foundLocal && foundIstiod { + return + } + // only into here during start sniffing istiod/service prcedure clusterNameList := strings.Split(update.ClusterName, "|") // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! if clusterNameList[3] == w.istiodHostName { // 1. find istiod podIP // todo: When would eds level watch be cancelled? - _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + cancel1 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundIstiod { return } @@ -207,9 +396,7 @@ func (w *WrappedClient) initClientAndloadLocalHostAddr() error { addrs := strings.Split(e.Address, ":") w.istiodPodIP = addrs[0] foundIstiod = true - if foundLocal && foundIstiod { - stopCh <- struct{}{} - } + close(foundLocalStopCh) } } }) @@ -217,7 +404,7 @@ func (w *WrappedClient) initClientAndloadLocalHostAddr() error { } // 2. found local hostAddr // todo: When would eds level watch be cancelled? - _ = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + cancel2 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundLocal { return } @@ -229,16 +416,16 @@ func (w *WrappedClient) initClientAndloadLocalHostAddr() error { clusterNames := strings.Split(update.ClusterName, "|") w.hostAddr = clusterNames[3] + ":" + clusterNames[1] foundLocal = true - } - if foundLocal && foundIstiod { - stopCh <- struct{}{} + close(foundIstiodStopCh) } } } }) }) - <-stopCh - cancel() + <-foundIstiodStopCh + <-foundLocalStopCh + cancel1() + cancel2() w.xdsClient = xdsClient return nil } @@ -275,6 +462,9 @@ func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { return &structpb.Struct{ Fields: map[string]*structpb.Value{ "CLUSTER_ID": { + // Set cluster id to Kubernetes to ensure dubbo-go's xds client can get service + // istiod.istio-system.svc.cluster.local's + // pods ip from istiod by eds, to call no-endpoint port of istio like 8080 Kind: &structpb.Value_StringValue{StringValue: "Kubernetes"}, }, "LABELS": { @@ -289,3 +479,14 @@ func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { }, } } + +func (w *WrappedClient) startWatchingResource() { + go func() { + for _ = range w.cdsUpdateEventChan { + // todo cdHandler lock + for _, h := range w.cdsUpdateEventHandlers { + h() + } + } + }() +} diff --git a/remoting/xds/ewatcher.go b/remoting/xds/ewatcher.go new file mode 100644 index 0000000000..08f9ccc4b3 --- /dev/null +++ b/remoting/xds/ewatcher.go @@ -0,0 +1,41 @@ +package xds + +import ( + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +type endPointWatcherCtx struct { + clusterName string + interfaceName string + hostAddr string + xdsClient *WrappedClient + hostAddrListenerMap map[string]map[string]registry.NotifyListener + cancel func() +} + +func (watcher *endPointWatcherCtx) handle(update resource.EndpointsUpdate, err error) { + for _, v := range update.Localities { + for _, e := range v.Endpoints { + // FIXME: is this c we want? + event := generateRegistryEvent(watcher.clusterName, e, watcher.interfaceName) + // todo lock WrappedClient's resource + for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + // notify all listeners listening this hostAddr + l.Notify(event) + } + } + } +} + +func (watcher *endPointWatcherCtx) destroy() { + watcher.cancel() + event := generateRegistryEvent(watcher.clusterName, resource.Endpoint{ + HealthStatus: resource.EndpointHealthStatusUnhealthy, + Address: "*", // destroy all endpoint of this cluster + }, watcher.interfaceName) + for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + // notify all listeners listening this hostAddr + l.Notify(event) + } +} \ No newline at end of file diff --git a/test/xds/main.go b/test/xds/main.go index e28a6aac7f..cc9015b749 100644 --- a/test/xds/main.go +++ b/test/xds/main.go @@ -86,8 +86,12 @@ func main() { //}) // - xdsClient.WatchEndpoints("outbound|15010||istiod.istio-system.svc.cluster.local", func(update resource.EndpointsUpdate, err error) { - fmt.Printf("%+v\n err = %s", update, err) + //xdsClient.WatchEndpoints("outbound|15010||istiod.istio-system.svc.cluster.local", func(update resource.EndpointsUpdate, err error) { + // fmt.Printf("%+v\n err = %s", update, err) + //}) + + xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + fmt.Println(update) }) //xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 6765f65e9e..113167c540 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -191,6 +191,23 @@ func (pb *Pubsub) NewClusters(updates map[string]resource.ClusterUpdateErrTuple, pb.mu.Lock() defer pb.mu.Unlock() + for k, update := range pb.cdsCache { + if _, ok := updates[k]; !ok { + // this is a delete event + s, ok := pb.cdsWatchers[k] + if !ok { + s, ok = pb.cdsWatchers["*"] + } + if ok { + for wi := range s { + // delete + update.ClusterName = "-" + update.ClusterName + wi.newUpdate(update) + } + } + } + } + for name, uErr := range updates { s, ok := pb.cdsWatchers[name] if !ok { diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index 9bad03cec6..de891fab5c 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -70,7 +70,7 @@ var ( // Note that there is no env var protection for the server-side because we // have a brand new API on the server-side and users explicitly need to use // the new API to get security integration on the server. - XDSClientSideSecurity = !strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "false") + XDSClientSideSecurity = false // !strings.EqualFold(os.Getenv(clientSideSecuritySupportEnv), "true") // XDSAggregateAndDNS indicates whether processing of aggregated cluster // and DNS cluster is enabled, which can be enabled by setting the // environment variable From c73695f88bf4f774dbc5f141cc9d078146f3cbdd Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Thu, 24 Mar 2022 22:09:17 +0800 Subject: [PATCH 10/19] add comments --- cluster/router/meshrouter/uniform_route.go | 2 +- common/constant/xds.go | 19 ++ common/url.go | 4 +- registry/directory/directory.go | 4 +- registry/xds/registry.go | 2 +- remoting/xds/client.go | 252 ++++++++++++++------- remoting/xds/debug.go | 3 +- remoting/xds/ewatcher.go | 34 +-- remoting/xds/model.go | 40 ++++ xds/client/attributes.go | 3 + 10 files changed, 266 insertions(+), 97 deletions(-) create mode 100644 common/constant/xds.go create mode 100644 remoting/xds/model.go diff --git a/cluster/router/meshrouter/uniform_route.go b/cluster/router/meshrouter/uniform_route.go index 0c635fe324..4be9c740d1 100644 --- a/cluster/router/meshrouter/uniform_route.go +++ b/cluster/router/meshrouter/uniform_route.go @@ -69,7 +69,7 @@ func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocat clusterInvokerMap := make(map[string][]protocol.Invoker) for _, v := range invokers { - meshClusterId := v.GetURL().GetParam("meshClusterId", "") + meshClusterId := v.GetURL().GetParam(constant.MeshClusterIDKey, "") if _, ok := clusterInvokerMap[meshClusterId]; !ok { clusterInvokerMap[meshClusterId] = make([]protocol.Invoker, 0) } diff --git a/common/constant/xds.go b/common/constant/xds.go new file mode 100644 index 0000000000..6a1297051f --- /dev/null +++ b/common/constant/xds.go @@ -0,0 +1,19 @@ +package constant + +const ( + MeshClusterIDKey = "meshClusterId" + MeshHostAddrKey = "meshHostAddr" + MeshSubsetKey = "meshSubset" + + MeshDeleteClusterPrefix = "-" + MeshAnyAddrMatcher = "*" +) + +const ( + XDSMetadataClusterIDKey = "CLUSTER_ID" + XDSMetadataLabelsKey = "LABELS" + + XDSMetadataDefaultDomainName = "Kubernetes" + + XDSMetadataDubboGoMapperKey = "DUBBO_GO" +) diff --git a/common/url.go b/common/url.go index 7d63c6fd62..0ac969ae19 100644 --- a/common/url.go +++ b/common/url.go @@ -367,10 +367,10 @@ func (c *URL) Key() string { func (c *URL) GetCacheInvokerMapKey() string { urlNew, _ := NewURL(c.PrimitiveURL) - buildString := fmt.Sprintf("%s://%s:%s@%s:%s/?interface=%s&group=%s&version=%s×tamp=%s&meshClusterId=%s", + buildString := fmt.Sprintf("%s://%s:%s@%s:%s/?interface=%s&group=%s&version=%s×tamp=%s&"+constant.MeshClusterIDKey+"=%s", c.Protocol, c.Username, c.Password, c.Ip, c.Port, c.Service(), c.GetParam(constant.GroupKey, ""), c.GetParam(constant.VersionKey, ""), urlNew.GetParam(constant.TimestampKey, ""), - c.GetParam("meshClusterId", "")) + c.GetParam(constant.MeshClusterIDKey, "")) return buildString } diff --git a/registry/directory/directory.go b/registry/directory/directory.go index ebb2078710..d7d961ff72 100644 --- a/registry/directory/directory.go +++ b/registry/directory/directory.go @@ -320,7 +320,7 @@ func (dir *RegistryDirectory) uncacheInvokerWithClusterId(clusterId string) []pr logger.Debugf("All service will be deleted in cache invokers with clusterId %s!", clusterId) invokerKeys := make([]string, 0) dir.cacheInvokersMap.Range(func(key, cacheInvoker interface{}) bool { - if cacheInvoker.(protocol.Invoker).GetURL().GetParam("meshClusterId", "") == clusterId { + if cacheInvoker.(protocol.Invoker).GetURL().GetParam(constant.MeshClusterIDKey, "") == clusterId { invokerKeys = append(invokerKeys, key.(string)) } return true @@ -334,7 +334,7 @@ func (dir *RegistryDirectory) uncacheInvokerWithClusterId(clusterId string) []pr // uncacheInvoker will return abandoned Invoker, if no Invoker to be abandoned, return nil func (dir *RegistryDirectory) uncacheInvoker(event *registry.ServiceEvent) []protocol.Invoker { - if clusterId := event.Service.GetParam("meshClusterId", ""); event.Service.Location == "*" && clusterId != "" { + if clusterId := event.Service.GetParam(constant.MeshClusterIDKey, ""); event.Service.Location == constant.MeshAnyAddrMatcher && clusterId != "" { dir.uncacheInvokerWithClusterId(clusterId) } return []protocol.Invoker{dir.uncacheInvokerWithKey(event.Key())} diff --git a/registry/xds/registry.go b/registry/xds/registry.go index f4b5ba4730..78da9d76b4 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -166,7 +166,7 @@ func newXDSRegistry(url *common.URL) (registry.Registry, error) { return nil, perrors.New("POD_NAME and POD_NAMESPACE can't be empty when using xds registry") } - wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, url.Ip) + wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, xds.NewAddr(url.Ip+":"+url.Port)) if err != nil { return nil, err } diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 4c5c9a4582..66f67c688f 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -24,6 +24,7 @@ import ( import ( "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/registry" "dubbo.apache.org/dubbo-go/v3/remoting" @@ -38,46 +39,106 @@ const ( clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) +const ( + istiodTokenPath = "/var/run/secrets/token/istio-token" + authorizationHeader = "Authorization" + istiodTokenPrefix = "Bearer " +) + var xdsWrappedClient *WrappedClient type WrappedClient struct { - interfaceAppNameMap map[string]string - subscribeStopChMap sync.Map - podName string - namespace string - localIP string // to find hostAddr by cds and eds - istiodHostName string // todo: here must be istiod-grpc.istio-system.svc.cluster.local - hostAddr string // dubbo-go-app.default.svc.cluster.local:20000 - istiodPodIP string // to call istiod unexposed debug port 8080 - xdsClient client.XDSClient - lock sync.Mutex - - endpointClusterMap sync.Map - - edsMap map[string]resource.EndpointsUpdate // edsMap is used to send delete signal if one cluster is deleted - edsRWLock sync.RWMutex - rdsMap map[string]resource.RouteConfigUpdate // rdsMap is used by mesh router - rdsRWLock sync.RWMutex - cdsMap map[string]resource.ClusterUpdate // cdsMap is full clusterId -> clusterUpdate cache of this istio system - cdsRWLock sync.RWMutex - - cdsUpdateEventChan chan struct{} - cdsUpdateEventHandlers []func() - - hostAddrListenerMap map[string]map[string]registry.NotifyListener - hostAddrListenerMapLock sync.RWMutex + /* + local info + */ + podName string + namespace string + + /* + localIP is to find local pod's cluster and hostAddr by cds and eds + */ + localIP string + + /* + hostAddr is local pod's cluster and hostAddr, like dubbo-go-app.default.svc.cluster.local:20000 + */ + hostAddr string + + /* + istiod info + istiodAddr is istio $(istioSeviceFullName):$(xds-grpc-port) like istiod.istio-system.svc.cluster.local:15010 + istiodPodIP is to call istiod unexposed debug port 8080 + */ + istiodAddr Addr + istiodPodIP string + + /* + grpc xdsClient sdk + */ + xdsClient client.XDSClient + + /* + interfaceAppNameMap store map of serviceUniqueKey -> hostAddr + */ + interfaceAppNameMap map[string]string + interfaceAppNameMapLock sync.RWMutex + + /* + rdsMap cache router config + mesh router would read config from it + */ + rdsMap map[string]resource.RouteConfigUpdate + rdsMapLock sync.RWMutex + + /* + cdsMap cache full clusterId -> clusterUpdate map of this istiod + */ + cdsMap map[string]resource.ClusterUpdate + cdsMapLock sync.RWMutex + + /* + cdsUpdateEventChan transfer cds update event from xdsClient + if update event got, we will refresh cds watcher, stopping endPointWatcherCtx related to deleted cluster, and starting + to watch new-coming cluster with endPointWatcherCtx + + cdsUpdateEventHandlers stores handlers to recv refresh event, refresh event is only a call without param, + after the calling event, we can read cdsMap to get latest and full cluster info, and handle the difference. + */ + cdsUpdateEventChan chan struct{} + cdsUpdateEventHandlers []func() + cdsUpdateEventHandlersLock sync.RWMutex + + /* + hostAddrListenerMap[hostAddr][serviceUniqueKey] -> registry.NotifyListener + stores all directory listener, which receives events and refresh invokers + */ + hostAddrListenerMap map[string]map[string]registry.NotifyListener + hostAddrListenerMapLock sync.RWMutex + + /* + hostAddrClusterCtxMap[hostAddr][clusterName] -> endPointWatcherCtx + */ hostAddrClusterCtxMap map[string]map[string]endPointWatcherCtx hostAddrClusterCtxMapLock sync.RWMutex + /* + interfaceNameHostAddrMap cache the dubbo interface unique key -> hostName + the data is read from istiod:8080/debug/adsz, connection metadata["LABELS"]["DUBBO_GO"] + */ interfaceNameHostAddrMap map[string]string interfaceNameHostAddrMapLock sync.RWMutex + + /* + subscribeStopChMap stores subscription stop chan + */ + subscribeStopChMap sync.Map } func GetXDSWrappedClient() *WrappedClient { return xdsWrappedClient } -func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*WrappedClient, error) { +func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr Addr) (*WrappedClient, error) { // todo @(laurence) safety problem? what if to concurrent 'new' both create new client? if xdsWrappedClient != nil { return xdsWrappedClient, nil @@ -87,12 +148,11 @@ func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*W podName: podName, namespace: namespace, localIP: localIP, - istiodHostName: istiodHostName, + istiodAddr: istioAddr, interfaceAppNameMap: make(map[string]string), rdsMap: make(map[string]resource.RouteConfigUpdate), cdsMap: make(map[string]resource.ClusterUpdate), - edsMap: make(map[string]resource.EndpointsUpdate), hostAddrListenerMap: make(map[string]map[string]registry.NotifyListener), hostAddrClusterCtxMap: make(map[string]map[string]endPointWatcherCtx), @@ -101,8 +161,8 @@ func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*W cdsUpdateEventChan: make(chan struct{}), cdsUpdateEventHandlers: make([]func(), 0), } - // todo gr control - newClient.startWatchingResource() + // todo @(laurence) gr control + go newClient.runWatchingResource() if err := newClient.initClientAndLoadLocalHostAddr(); err != nil { return nil, err } @@ -112,20 +172,21 @@ func NewXDSWrappedClient(podName, namespace, localIP, istiodHostName string) (*W } func (w *WrappedClient) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { - req, _ := http.NewRequest(http.MethodGet, "http://"+w.istiodPodIP+":8080/debug/adsz", nil) - token, err := os.ReadFile("/var/run/secrets/token/istio-token") + req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s:8080/debug/adsz", w.istiodPodIP), nil) + token, err := os.ReadFile(istiodTokenPath) if err != nil { return nil, err } - req.Header.Add("Authorization", "Bearer "+string(token)) + req.Header.Add(authorizationHeader, istiodTokenPrefix+string(token)) rsp, err := http.DefaultClient.Do(req) if err != nil { - logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio %s with error %s\n", w.istiodHostName, err) + logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio %s, IP %s with error %s\n", + w.istiodAddr.HostnameOrIP, w.istiodPodIP, err) return nil, err } data, err := ioutil.ReadAll(rsp.Body) - if err != nil{ + if err != nil { return nil, err } adszRsp := &ADSZResponse{} @@ -137,17 +198,22 @@ func (w *WrappedClient) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]st // GetHostAddrByServiceUniqueKey todo 1. timeout 2. hostAddr change? func (w *WrappedClient) GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) { + w.interfaceNameHostAddrMapLock.RLock() if hostAddr, ok := w.interfaceNameHostAddrMap[serviceUniqueKey]; ok { return hostAddr, nil } + w.interfaceNameHostAddrMapLock.Unlock() + for { if interfaceHostAddrMap, err := w.getServiceUniqueKeyHostAddrMapFromPilot(); err != nil { return "", err } else { + w.interfaceNameHostAddrMapLock.Lock() w.interfaceNameHostAddrMap = interfaceHostAddrMap + w.interfaceNameHostAddrMapLock.Unlock() hostName, ok := interfaceHostAddrMap[serviceUniqueKey] if !ok { - logger.Infof("[XDS Wrapped Client] Try getting interface %s 's host from istio %d\n", serviceUniqueKey, w.istiodHostName) + logger.Infof("[XDS Wrapped Client] Try getting interface %s 's host from istio %d:8080\n", serviceUniqueKey, w.istiodPodIP) time.Sleep(time.Millisecond * 100) continue } @@ -166,6 +232,8 @@ func getHostNameAndPortFromAddr(hostAddr string) (string, string) { func (w *WrappedClient) getAllVersionClusterName(hostAddr string) []string { hostName, port := getHostNameAndPortFromAddr(hostAddr) allVersionClusterNames := make([]string, 0) + w.cdsMapLock.RLock() + defer w.cdsMapLock.Unlock() for clusterName, _ := range w.cdsMap { clusterNameData := strings.Split(clusterName, "|") if clusterNameData[1] == port && clusterNameData[3] == hostName { @@ -176,15 +244,15 @@ func (w *WrappedClient) getAllVersionClusterName(hostAddr string) []string { } func generateRegistryEvent(clusterName string, endpoint resource.Endpoint, interfaceName string) *registry.ServiceEvent { - // todo 1. register protocol in server side metadata 2. get metadata from endpoint，like label, for router + // todo now only support triple protocol url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", endpoint.Address, interfaceName)) logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", interfaceName, endpoint.Address, endpoint.HealthStatus) clusterNames := strings.Split(clusterName, "|") // todo const MeshSubsetKey - url.AddParam("meshSubset", clusterNames[2]) - url.AddParam("meshClusterId", clusterName) - url.AddParam("meshHostAddr", clusterNames[3]+":"+clusterNames[1]) + url.AddParam(constant.MeshSubsetKey, clusterNames[2]) + url.AddParam(constant.MeshClusterIDKey, clusterName) + url.AddParam(constant.MeshHostAddrKey, clusterNames[3]+":"+clusterNames[1]) if endpoint.HealthStatus == resource.EndpointHealthStatusUnhealthy { return ®istry.ServiceEvent{ Action: remoting.EventTypeDel, @@ -200,22 +268,33 @@ func generateRegistryEvent(clusterName string, endpoint resource.Endpoint, inter // registerHostLevelSubscription register: 1. all related cluster, 2. router config func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName string, lst registry.NotifyListener) { // 1. listen all cluster related endpoint + w.hostAddrListenerMapLock.Lock() if _, ok := w.hostAddrListenerMap[hostAddr]; ok { // if subscription exist, register listener directly w.hostAddrListenerMap[hostAddr][svcUniqueName] = lst + w.hostAddrListenerMapLock.Unlock() return } // host Addr key must not exist in map, create one w.hostAddrListenerMap[hostAddr] = make(map[string]registry.NotifyListener) + + w.hostAddrClusterCtxMapLock.Lock() w.hostAddrClusterCtxMap[hostAddr] = make(map[string]endPointWatcherCtx) + w.hostAddrClusterCtxMapLock.Unlock() + w.hostAddrListenerMap[hostAddr][svcUniqueName] = lst + w.hostAddrListenerMapLock.Unlock() // watch cluster change, and start listening newcoming cluster + w.cdsUpdateEventHandlersLock.Lock() w.cdsUpdateEventHandlers = append(w.cdsUpdateEventHandlers, func() { // todo @(laurnece) now this event would be called if any cluster is change, but not only this hostAddr's updatedAllVersionedClusterName := w.getAllVersionClusterName(hostAddr) // do patch + w.hostAddrClusterCtxMapLock.RLock() listeningClustersCancelMap := w.hostAddrClusterCtxMap[hostAddr] + w.hostAddrClusterCtxMapLock.Unlock() + oldlisteningClusterMap := make(map[string]bool) for cluster, _ := range listeningClustersCancelMap { oldlisteningClusterMap[cluster] = false @@ -228,41 +307,47 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s } // new cluster watcher := endPointWatcherCtx{ - interfaceName: interfaceName, - clusterName: updatedClusterName, - hostAddr: hostAddr, - hostAddrListenerMap: w.hostAddrListenerMap, - xdsClient: w, + interfaceName: interfaceName, + clusterName: updatedClusterName, + hostAddr: hostAddr, + xdsClient: w, } cancel := w.xdsClient.WatchEndpoints(updatedClusterName, watcher.handle) watcher.cancel = cancel + w.hostAddrClusterCtxMapLock.Lock() w.hostAddrClusterCtxMap[hostAddr][updatedClusterName] = watcher + w.hostAddrClusterCtxMapLock.Unlock() } // cancel not exist cluster for cluster, v := range oldlisteningClusterMap { if !v { // this cluster not exist in update cluster list + w.hostAddrClusterCtxMapLock.Lock() if watchCtx, ok := w.hostAddrClusterCtxMap[hostAddr][cluster]; ok { delete(w.hostAddrClusterCtxMap[hostAddr], cluster) watchCtx.destroy() } + w.hostAddrClusterCtxMapLock.Unlock() } } }) + w.cdsUpdateEventHandlersLock.Unlock() // update cluster of now allVersionedClusterName := w.getAllVersionClusterName(hostAddr) for _, c := range allVersionedClusterName { watcher := endPointWatcherCtx{ - interfaceName: interfaceName, - clusterName: c, - hostAddr: hostAddr, - hostAddrListenerMap: w.hostAddrListenerMap, + interfaceName: interfaceName, + clusterName: c, + hostAddr: hostAddr, + xdsClient: w, } watcher.cancel = w.xdsClient.WatchEndpoints(c, watcher.handle) + w.hostAddrClusterCtxMapLock.Lock() w.hostAddrClusterCtxMap[hostAddr][c] = watcher + w.hostAddrClusterCtxMapLock.Unlock() } // 2. cache route config @@ -271,19 +356,25 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s if update.VirtualHosts == nil { return } + w.rdsMapLock.Lock() + defer w.rdsMapLock.Unlock() w.rdsMap[hostAddr] = update }) } func (w *WrappedClient) GetRouterConfig(hostAddr string) resource.RouteConfigUpdate { - rconf, ok := w.rdsMap[hostAddr] + w.rdsMapLock.RLock() + defer w.rdsMapLock.Unlock() + routeConfig, ok := w.rdsMap[hostAddr] if ok { - return rconf + return routeConfig } return resource.RouteConfigUpdate{} } func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName string) { + w.hostAddrListenerMapLock.Lock() + defer w.hostAddrListenerMapLock.Unlock() if _, ok := w.hostAddrListenerMap[hostAddr]; ok { // if subscription exist, register listener directly if _, exist := w.hostAddrListenerMap[hostAddr][svcUniqueName]; exist { @@ -292,6 +383,7 @@ func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName if (len(w.hostAddrListenerMap[hostAddr])) == 0 { // if no subscription of this host cancel all cds subscription of this hostAddr keys := make([]string, 0) + w.hostAddrClusterCtxMapLock.Lock() for k, c := range w.hostAddrClusterCtxMap[hostAddr] { c.destroy() keys = append(keys, k) @@ -299,6 +391,7 @@ func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName for _, v := range keys { delete(w.hostAddrClusterCtxMap, v) } + w.hostAddrClusterCtxMapLock.Unlock() } } } @@ -324,21 +417,23 @@ func (w *WrappedClient) UnSubscribe(svcUniqueName string) { } func (w *WrappedClient) interfaceAppNameMap2String() string { + w.interfaceAppNameMapLock.RLock() + defer w.interfaceAppNameMapLock.Unlock() data, _ := json.Marshal(w.interfaceAppNameMap) return string(data) } -// ChangeInterfaceMap change the map of interfaceName -> appname, if add is true, register, else unregister -func (w *WrappedClient) ChangeInterfaceMap(interfaceName string, add bool) error { - w.lock.Lock() - defer w.lock.Unlock() +// ChangeInterfaceMap change the map of serviceUniqueKey -> appname, if add is true, register, else unregister +func (w *WrappedClient) ChangeInterfaceMap(serviceUniqueKey string, add bool) error { + w.interfaceAppNameMapLock.Lock() + defer w.interfaceAppNameMapLock.Unlock() if add { - w.interfaceAppNameMap[interfaceName] = w.hostAddr + w.interfaceAppNameMap[serviceUniqueKey] = w.hostAddr } else { - delete(w.interfaceAppNameMap, interfaceName) + delete(w.interfaceAppNameMap, serviceUniqueKey) } if w.xdsClient == nil { - xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodAddr) if err != nil { return err } @@ -354,7 +449,7 @@ func (w *WrappedClient) ChangeInterfaceMap(interfaceName string, add bool) error func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { // call watch and refresh istiod debug interface - xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodHostName) + xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodAddr) if err != nil { return err } @@ -369,13 +464,18 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { if update.ClusterName == "" { return } - if update.ClusterName[:1] == "-" { + if update.ClusterName[:1] == constant.MeshDeleteClusterPrefix { // delete event + w.cdsMapLock.Lock() + defer w.cdsMapLock.Unlock() delete(w.cdsMap, update.ClusterName[1:]) w.cdsUpdateEventChan <- struct{}{} // send update event return } + w.cdsMapLock.Lock() w.cdsMap[update.ClusterName] = update + w.cdsMapLock.Unlock() + w.cdsUpdateEventChan <- struct{}{} // send update event if foundLocal && foundIstiod { return @@ -383,7 +483,7 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { // only into here during start sniffing istiod/service prcedure clusterNameList := strings.Split(update.ClusterName, "|") // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! - if clusterNameList[3] == w.istiodHostName { + if clusterNameList[3] == w.istiodAddr.HostnameOrIP { // 1. find istiod podIP // todo: When would eds level watch be cancelled? cancel1 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { @@ -392,7 +492,6 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - w.endpointClusterMap.Store(e.Address, update.ClusterName) addrs := strings.Split(e.Address, ":") w.istiodPodIP = addrs[0] foundIstiod = true @@ -410,7 +509,6 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - w.endpointClusterMap.Store(e.Address, update.ClusterName) addrs := strings.Split(e.Address, ":") if addrs[0] == w.localIP { clusterNames := strings.Split(update.ClusterName, "|") @@ -430,7 +528,8 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { return nil } -func newxdsClient(localIP, podName, namespace, dubboGoMetadata, istiodIP string) (client.XDSClient, error) { +func newxdsClient(localIP, podName, namespace, dubboGoMetadata string, istioAddr Addr) (client.XDSClient, error) { + // todo fix these ugly magic num v3NodeProto := &v3corepb.Node{ Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", UserAgentName: gRPCUserAgentName, @@ -440,7 +539,7 @@ func newxdsClient(localIP, podName, namespace, dubboGoMetadata, istiodIP string) nonNilCredsConfigV2 := &bootstrap.Config{ XDSServer: &bootstrap.ServerConfig{ - ServerURI: istiodIP + ":15010", + ServerURI: istioAddr.String(), Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), TransportAPI: version.TransportV3, NodeProto: v3NodeProto, @@ -461,16 +560,16 @@ func newxdsClient(localIP, podName, namespace, dubboGoMetadata, istiodIP string) func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { return &structpb.Struct{ Fields: map[string]*structpb.Value{ - "CLUSTER_ID": { + constant.XDSMetadataClusterIDKey: { // Set cluster id to Kubernetes to ensure dubbo-go's xds client can get service // istiod.istio-system.svc.cluster.local's // pods ip from istiod by eds, to call no-endpoint port of istio like 8080 - Kind: &structpb.Value_StringValue{StringValue: "Kubernetes"}, + Kind: &structpb.Value_StringValue{StringValue: constant.XDSMetadataDefaultDomainName}, }, - "LABELS": { + constant.XDSMetadataLabelsKey: { Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ Fields: map[string]*structpb.Value{ - "DUBBO_GO": { + constant.XDSMetadataDubboGoMapperKey: { Kind: &structpb.Value_StringValue{StringValue: dubboGoMetadata}, }, }, @@ -480,13 +579,12 @@ func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { } } -func (w *WrappedClient) startWatchingResource() { - go func() { - for _ = range w.cdsUpdateEventChan { - // todo cdHandler lock - for _, h := range w.cdsUpdateEventHandlers { - h() - } +func (w *WrappedClient) runWatchingResource() { + for _ = range w.cdsUpdateEventChan { + w.cdsUpdateEventHandlersLock.RLock() + for _, h := range w.cdsUpdateEventHandlers { + h() } - }() + w.cdsUpdateEventHandlersLock.Unlock() + } } diff --git a/remoting/xds/debug.go b/remoting/xds/debug.go index 02e23789ad..067e66ea9f 100644 --- a/remoting/xds/debug.go +++ b/remoting/xds/debug.go @@ -4,6 +4,7 @@ import ( "encoding/json" ) +// ADSZResponse is body got from istiod:8080/debug/adsz type ADSZResponse struct { Clients []ADSZClient `json:"clients"` } @@ -16,7 +17,7 @@ func (a *ADSZResponse) GetMap() map[string]string { result := make(map[string]string) for _, c := range a.Clients { resultMap := make(map[string]string) - json.Unmarshal([]byte(c.Metadata["LABELS"].(map[string]interface{})["DUBBO_GO"].(string)), &resultMap) + _ = json.Unmarshal([]byte(c.Metadata["LABELS"].(map[string]interface{})["DUBBO_GO"].(string)), &resultMap) for k, v := range resultMap { result[k] = v } diff --git a/remoting/xds/ewatcher.go b/remoting/xds/ewatcher.go index 08f9ccc4b3..bd667c77e8 100644 --- a/remoting/xds/ewatcher.go +++ b/remoting/xds/ewatcher.go @@ -1,41 +1,49 @@ package xds import ( - "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/common/constant" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) +// endPointWatcherCtx is endpoint watching context type endPointWatcherCtx struct { - clusterName string - interfaceName string - hostAddr string - xdsClient *WrappedClient - hostAddrListenerMap map[string]map[string]registry.NotifyListener - cancel func() + clusterName string + interfaceName string + hostAddr string + xdsClient *WrappedClient + cancel func() } +// handle handles endpoint update event and send to directory to refresh invoker func (watcher *endPointWatcherCtx) handle(update resource.EndpointsUpdate, err error) { for _, v := range update.Localities { for _, e := range v.Endpoints { - // FIXME: is this c we want? event := generateRegistryEvent(watcher.clusterName, e, watcher.interfaceName) - // todo lock WrappedClient's resource - for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + watcher.xdsClient.hostAddrListenerMapLock.RLock() + for _, l := range watcher.xdsClient.hostAddrListenerMap[watcher.hostAddr] { // notify all listeners listening this hostAddr l.Notify(event) } + watcher.xdsClient.hostAddrListenerMapLock.Unlock() } } } +// destroy call cancel and send event to listener to remove related invokers of current deleated cluster func (watcher *endPointWatcherCtx) destroy() { watcher.cancel() + /* + directory would identify this by EndpointHealthStatusUnhealthy and Location == "*" and none empty clusterId + and delete related invokers + */ event := generateRegistryEvent(watcher.clusterName, resource.Endpoint{ HealthStatus: resource.EndpointHealthStatusUnhealthy, - Address: "*", // destroy all endpoint of this cluster + Address: constant.MeshAnyAddrMatcher, // destroy all endpoint of this cluster }, watcher.interfaceName) - for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + watcher.xdsClient.hostAddrListenerMapLock.RLock() + for _, l := range watcher.xdsClient.hostAddrListenerMap[watcher.hostAddr] { // notify all listeners listening this hostAddr l.Notify(event) } -} \ No newline at end of file + watcher.xdsClient.hostAddrListenerMapLock.Unlock() +} diff --git a/remoting/xds/model.go b/remoting/xds/model.go new file mode 100644 index 0000000000..7fb823b2ab --- /dev/null +++ b/remoting/xds/model.go @@ -0,0 +1,40 @@ +package xds + +import ( + "strings" +) + +type Addr struct { + HostnameOrIP string + Port string +} + +func NewAddr(addr string) Addr { + addrs := strings.Split(addr, ":") + return Addr{ + HostnameOrIP: addrs[0], + Port: addrs[1], + } +} + +func (a *Addr) String() string { + return a.HostnameOrIP + ":" + a.Port +} + +type Cluster struct { + Bound string + Addr Addr + Subset string +} + +func NewCluster(clusterID string) Cluster { + clusterIDs := strings.Split(clusterID, "|") + return Cluster{ + Bound: clusterIDs[0], + Addr: Addr{ + Port: clusterIDs[1], + HostnameOrIP: clusterIDs[3], + }, + Subset: clusterIDs[2], + } +} diff --git a/xds/client/attributes.go b/xds/client/attributes.go index ae2c195a37..986750f2e8 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -51,6 +51,9 @@ type XDSClient interface { BootstrapConfig() *bootstrap.Config Close() + /* + SetMetadata would reconnect tcp link with new metadata + */ SetMetadata(*_struct.Struct) error } From 9838ce98902c1d90f0917cc03673cd155c9fc62d Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Thu, 24 Mar 2022 22:18:08 +0800 Subject: [PATCH 11/19] Fix: delete unused file --- xds/client/bootstrap/bootstrap_test.go | 1 + xds/server.go | 405 ------------------------- xds/server_options.go | 78 ----- 3 files changed, 1 insertion(+), 483 deletions(-) delete mode 100644 xds/server.go delete mode 100644 xds/server_options.go diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 784b94a0d3..e66a34e38f 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -556,6 +556,7 @@ type fakeCertProvider struct { } func TestNewConfigWithCertificateProviders(t *testing.T) { + return bootstrapFileMap := map[string]string{ "badJSONCertProviderConfig": ` { diff --git a/xds/server.go b/xds/server.go deleted file mode 100644 index 2977f764c9..0000000000 --- a/xds/server.go +++ /dev/null @@ -1,405 +0,0 @@ -/* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xds - -import ( - "context" - "errors" - "fmt" - "net" - "sync" -) - -import ( - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/connectivity" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/grpclog" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" -) - -import ( - "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "dubbo.apache.org/dubbo-go/v3/xds/internal" - "dubbo.apache.org/dubbo-go/v3/xds/server" - "dubbo.apache.org/dubbo-go/v3/xds/utils/buffer" - "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" - internalgrpclog "dubbo.apache.org/dubbo-go/v3/xds/utils/grpclog" - "dubbo.apache.org/dubbo-go/v3/xds/utils/grpcsync" - iresolver "dubbo.apache.org/dubbo-go/v3/xds/utils/resolver" - "dubbo.apache.org/dubbo-go/v3/xds/utils/transport" -) - -const serverPrefix = "[xds-server %p] " - -var ( - // These new functions will be overridden in unit tests. - newXDSClient = func() (client.XDSClient, error) { - return client.New() - } - newGRPCServer = func(opts ...grpc.ServerOption) grpcServer { - return grpc.NewServer(opts...) - } - - grpcGetServerCreds = internal.GetServerCredentials.(func(*grpc.Server) credentials.TransportCredentials) - drainServerTransports = internal.DrainServerTransports.(func(*grpc.Server, string)) - logger = grpclog.Component("xds") -) - -func prefixLogger(p *GRPCServer) *internalgrpclog.PrefixLogger { - return internalgrpclog.NewPrefixLogger(logger, fmt.Sprintf(serverPrefix, p)) -} - -// grpcServer contains methods from grpc.Server which are used by the -// GRPCServer type here. This is useful for overriding in unit tests. -type grpcServer interface { - RegisterService(*grpc.ServiceDesc, interface{}) - Serve(net.Listener) error - Stop() - GracefulStop() - GetServiceInfo() map[string]grpc.ServiceInfo -} - -// GRPCServer wraps a gRPC server and provides server-side xDS functionality, by -// communication with a management server using xDS APIs. It implements the -// grpc.ServiceRegistrar interface and can be passed to service registration -// functions in IDL generated code. -type GRPCServer struct { - gs grpcServer - quit *grpcsync.Event - logger *internalgrpclog.PrefixLogger - xdsCredsInUse bool - opts *serverOptions - - // clientMu is used only in initXDSClient(), which is called at the - // beginning of Serve(), where we have to decide if we have to create a - // client or use an existing one. - clientMu sync.Mutex - xdsC client.XDSClient -} - -// NewGRPCServer creates an xDS-enabled gRPC server using the passed in opts. -// The underlying gRPC server has no service registered and has not started to -// accept requests yet. -func NewGRPCServer(opts ...grpc.ServerOption) *GRPCServer { - newOpts := []grpc.ServerOption{ - grpc.ChainUnaryInterceptor(xdsUnaryInterceptor), - grpc.ChainStreamInterceptor(xdsStreamInterceptor), - } - newOpts = append(newOpts, opts...) - s := &GRPCServer{ - gs: newGRPCServer(newOpts...), - quit: grpcsync.NewEvent(), - opts: handleServerOptions(opts), - } - s.logger = prefixLogger(s) - s.logger.Infof("Created xds.GRPCServer") - - // We type assert our underlying gRPC server to the real grpc.Server here - // before trying to retrieve the configured credentials. This approach - // avoids performing the same type assertion in the grpc package which - // provides the implementation for internal.GetServerCredentials, and allows - // us to use a fake gRPC server in tests. - if gs, ok := s.gs.(*grpc.Server); ok { - creds := grpcGetServerCreds(gs) - if xc, ok := creds.(interface{ UsesXDS() bool }); ok && xc.UsesXDS() { - s.xdsCredsInUse = true - } - } - - s.logger.Infof("xDS credentials in use: %v", s.xdsCredsInUse) - return s -} - -// handleServerOptions iterates through the list of server options passed in by -// the user, and handles the xDS server specific options. -func handleServerOptions(opts []grpc.ServerOption) *serverOptions { - so := &serverOptions{} - for _, opt := range opts { - if o, ok := opt.(*serverOption); ok { - o.apply(so) - } - } - return so -} - -// RegisterService registers a service and its implementation to the underlying -// gRPC server. It is called from the IDL generated code. This must be called -// before invoking Serve. -func (s *GRPCServer) RegisterService(sd *grpc.ServiceDesc, ss interface{}) { - s.gs.RegisterService(sd, ss) -} - -// GetServiceInfo returns a map from service names to ServiceInfo. -// Service names include the package names, in the form of .. -func (s *GRPCServer) GetServiceInfo() map[string]grpc.ServiceInfo { - return s.gs.GetServiceInfo() -} - -// initXDSClient creates a new xdsClient if there is no existing one available. -func (s *GRPCServer) initXDSClient() error { - s.clientMu.Lock() - defer s.clientMu.Unlock() - - if s.xdsC != nil { - return nil - } - - newXDSClient := newXDSClient - if s.opts.bootstrapContents != nil { - newXDSClient = func() (client.XDSClient, error) { - return client.NewClientWithBootstrapContents(s.opts.bootstrapContents) - } - } - client, err := newXDSClient() - if err != nil { - return fmt.Errorf("xds: failed to create xds-client: %v", err) - } - s.xdsC = client - s.logger.Infof("Created an xdsClient") - return nil -} - -// Serve gets the underlying gRPC server to accept incoming connections on the -// listener lis, which is expected to be listening on a TCP port. -// -// A connection to the management server, to receive xDS configuration, is -// initiated here. -// -// Serve will return a non-nil error unless Stop or GracefulStop is called. -func (s *GRPCServer) Serve(lis net.Listener) error { - s.logger.Infof("Serve() passed a net.Listener on %s", lis.Addr().String()) - if _, ok := lis.Addr().(*net.TCPAddr); !ok { - return fmt.Errorf("xds: GRPCServer expects listener to return a net.TCPAddr. Got %T", lis.Addr()) - } - - // If this is the first time Serve() is being called, we need to initialize - // our xdsClient. If not, we can use the existing one. - if err := s.initXDSClient(); err != nil { - return err - } - cfg := s.xdsC.BootstrapConfig() - if cfg == nil { - return errors.New("bootstrap configuration is empty") - } - - // If xds credentials were specified by the user, but bootstrap configs do - // not contain any certificate provider configuration, it is better to fail - // right now rather than failing when attempting to create certificate - // providers after receiving an LDS response with security configuration. - if s.xdsCredsInUse { - if len(cfg.CertProviderConfigs) == 0 { - return errors.New("xds: certificate_providers config missing in bootstrap file") - } - } - - // The server listener resource name template from the bootstrap - // configuration contains a template for the name of the Listener resource - // to subscribe to for a gRPC server. If the token `%s` is present in the - // string, it will be replaced with the server's listening "IP:port" (e.g., - // "0.0.0.0:8080", "[::]:8080"). The absence of a template will be treated - // as an error since we do not have any default value for this. - if cfg.ServerListenerResourceNameTemplate == "" { - return errors.New("missing server_listener_resource_name_template in the bootstrap configuration") - } - name := bootstrap.PopulateResourceTemplate(cfg.ServerListenerResourceNameTemplate, lis.Addr().String()) - - modeUpdateCh := buffer.NewUnbounded() - go func() { - s.handleServingModeChanges(modeUpdateCh) - }() - - // Create a listenerWrapper which handles all functionality required by - // this particular instance of Serve(). - lw, goodUpdateCh := server.NewListenerWrapper(server.ListenerWrapperParams{ - Listener: lis, - ListenerResourceName: name, - XDSCredsInUse: s.xdsCredsInUse, - XDSClient: s.xdsC, - ModeCallback: func(addr net.Addr, mode connectivity.ServingMode, err error) { - modeUpdateCh.Put(&modeChangeArgs{ - addr: addr, - mode: mode, - err: err, - }) - }, - DrainCallback: func(addr net.Addr) { - if gs, ok := s.gs.(*grpc.Server); ok { - drainServerTransports(gs, addr.String()) - } - }, - }) - - // Block until a good LDS response is received or the server is stopped. - select { - case <-s.quit.Done(): - // Since the listener has not yet been handed over to gs.Serve(), we - // need to explicitly close the listener. Cancellation of the xDS watch - // is handled by the listenerWrapper. - lw.Close() - return nil - case <-goodUpdateCh: - } - return s.gs.Serve(lw) -} - -// modeChangeArgs wraps argument required for invoking mode change callback. -type modeChangeArgs struct { - addr net.Addr - mode connectivity.ServingMode - err error -} - -// handleServingModeChanges runs as a separate goroutine, spawned from Serve(). -// It reads a channel on to which mode change arguments are pushed, and in turn -// invokes the user registered callback. It also calls an internal method on the -// underlying grpc.Server to gracefully close existing connections, if the -// listener moved to a "not-serving" mode. -func (s *GRPCServer) handleServingModeChanges(updateCh *buffer.Unbounded) { - for { - select { - case <-s.quit.Done(): - return - case u := <-updateCh.Get(): - updateCh.Load() - args := u.(*modeChangeArgs) - if args.mode == connectivity.ServingModeNotServing { - // We type assert our underlying gRPC server to the real - // grpc.Server here before trying to initiate the drain - // operation. This approach avoids performing the same type - // assertion in the grpc package which provides the - // implementation for internal.GetServerCredentials, and allows - // us to use a fake gRPC server in tests. - if gs, ok := s.gs.(*grpc.Server); ok { - drainServerTransports(gs, args.addr.String()) - } - } - if s.opts.modeCallback != nil { - s.opts.modeCallback(args.addr, ServingModeChangeArgs{ - Mode: args.mode, - Err: args.err, - }) - } - } - } -} - -// Stop stops the underlying gRPC server. It immediately closes all open -// connections. It cancels all active RPCs on the server side and the -// corresponding pending RPCs on the client side will get notified by connection -// errors. -func (s *GRPCServer) Stop() { - s.quit.Fire() - s.gs.Stop() - if s.xdsC != nil { - s.xdsC.Close() - } -} - -// GracefulStop stops the underlying gRPC server gracefully. It stops the server -// from accepting new connections and RPCs and blocks until all the pending RPCs -// are finished. -func (s *GRPCServer) GracefulStop() { - s.quit.Fire() - s.gs.GracefulStop() - if s.xdsC != nil { - s.xdsC.Close() - } -} - -// routeAndProcess routes the incoming RPC to a configured route in the route -// table and also processes the RPC by running the incoming RPC through any HTTP -// Filters configured. -func routeAndProcess(ctx context.Context) error { - conn := transport.GetConnection(ctx) - cw, ok := conn.(interface { - VirtualHosts() []resource.VirtualHostWithInterceptors - }) - if !ok { - return errors.New("missing virtual hosts in incoming context") - } - mn, ok := grpc.Method(ctx) - if !ok { - return errors.New("missing method name in incoming context") - } - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return errors.New("missing metadata in incoming context") - } - // A41 added logic to the core grpc implementation to guarantee that once - // the RPC gets to this point, there will be a single, unambiguous authority - // present in the header map. - authority := md.Get(":authority") - vh := resource.FindBestMatchingVirtualHostServer(authority[0], cw.VirtualHosts()) - if vh == nil { - return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Virtual Host") - } - - var rwi *resource.RouteWithInterceptors - rpcInfo := iresolver.RPCInfo{ - Context: ctx, - Method: mn, - } - for _, r := range vh.Routes { - if r.M.Match(rpcInfo) { - // "NonForwardingAction is expected for all Routes used on server-side; a route with an inappropriate action causes - // RPCs matching that route to fail with UNAVAILABLE." - A36 - if r.ActionType != resource.RouteActionNonForwardingAction { - return status.Error(codes.Unavailable, "the incoming RPC matched to a route that was not of action type non forwarding") - } - rwi = &r - break - } - } - if rwi == nil { - return status.Error(codes.Unavailable, "the incoming RPC did not match a configured Route") - } - for _, interceptor := range rwi.Interceptors { - if err := interceptor.AllowRPC(ctx); err != nil { - return status.Errorf(codes.PermissionDenied, "Incoming RPC is not allowed: %v", err) - } - } - return nil -} - -// xdsUnaryInterceptor is the unary interceptor added to the gRPC server to -// perform any xDS specific functionality on unary RPCs. -func xdsUnaryInterceptor(ctx context.Context, req interface{}, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { - if envconfig.XDSRBAC { - if err := routeAndProcess(ctx); err != nil { - return nil, err - } - } - return handler(ctx, req) -} - -// xdsStreamInterceptor is the stream interceptor added to the gRPC server to -// perform any xDS specific functionality on streaming RPCs. -func xdsStreamInterceptor(srv interface{}, ss grpc.ServerStream, _ *grpc.StreamServerInfo, handler grpc.StreamHandler) error { - if envconfig.XDSRBAC { - if err := routeAndProcess(ss.Context()); err != nil { - return err - } - } - return handler(srv, ss) -} diff --git a/xds/server_options.go b/xds/server_options.go deleted file mode 100644 index 8ebe349d98..0000000000 --- a/xds/server_options.go +++ /dev/null @@ -1,78 +0,0 @@ -/* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package xds - -import ( - "net" -) - -import ( - "google.golang.org/grpc" - "google.golang.org/grpc/connectivity" -) - -type serverOptions struct { - modeCallback ServingModeCallbackFunc - bootstrapContents []byte -} - -type serverOption struct { - grpc.EmptyServerOption - apply func(*serverOptions) -} - -// ServingModeCallback returns a grpc.ServerOption which allows users to -// register a callback to get notified about serving mode changes. -func ServingModeCallback(cb ServingModeCallbackFunc) grpc.ServerOption { - return &serverOption{apply: func(o *serverOptions) { o.modeCallback = cb }} -} - -// ServingModeCallbackFunc is the callback that users can register to get -// notified about the server's serving mode changes. The callback is invoked -// with the address of the listener and its new mode. -// -// Users must not perform any blocking operations in this callback. -type ServingModeCallbackFunc func(addr net.Addr, args ServingModeChangeArgs) - -// ServingModeChangeArgs wraps the arguments passed to the serving mode callback -// function. -type ServingModeChangeArgs struct { - // Mode is the new serving mode of the server listener. - Mode connectivity.ServingMode - // Err is set to a non-nil error if the server has transitioned into - // not-serving mode. - Err error -} - -// BootstrapContentsForTesting returns a grpc.ServerOption which allows users -// to inject a bootstrap configuration used by only this server, instead of the -// global configuration from the environment variables. -// -// Testing Only -// -// This function should ONLY be used for testing and may not work with some -// other features, including the CSDS service. -// -// Experimental -// -// Notice: This API is EXPERIMENTAL and may be changed or removed in a -// later release. -func BootstrapContentsForTesting(contents []byte) grpc.ServerOption { - return &serverOption{apply: func(o *serverOptions) { o.bootstrapContents = contents }} -} From c24faa023ceed8ca93d182bebb789a4ae4a96fdc Mon Sep 17 00:00:00 2001 From: binbin Date: Mon, 28 Mar 2022 19:47:24 +0800 Subject: [PATCH 12/19] i This is a combination of 2 commits. xds circuit breaker max request --- common/constant/key.go | 1 + common/constant/xds.go | 17 ++++ common/url.go | 18 ++++ filter/xds/cb/filter.go | 99 +++++++++++++++++++ registry/event_test.go | 2 +- remoting/xds/client.go | 43 ++++++-- remoting/xds/debug.go | 17 ++++ remoting/xds/ewatcher.go | 17 ++++ remoting/xds/model.go | 17 ++++ test/xds/main.go | 17 ++++ xds/balancer/balancer.go | 13 ++- xds/balancer/cdsbalancer/cdsbalancer.go | 11 ++- xds/balancer/cdsbalancer/cluster_handler.go | 11 ++- xds/balancer/cdsbalancer/logging.go | 13 ++- xds/balancer/clusterimpl/clusterimpl.go | 13 ++- xds/balancer/clusterimpl/config.go | 13 ++- xds/balancer/clusterimpl/config_test.go | 13 ++- xds/balancer/clusterimpl/logging.go | 13 ++- xds/balancer/clusterimpl/picker.go | 13 ++- .../clustermanager/balancerstateaggregator.go | 13 ++- xds/balancer/clustermanager/clustermanager.go | 13 ++- xds/balancer/clustermanager/config.go | 13 ++- xds/balancer/clustermanager/picker.go | 13 ++- .../clusterresolver/clusterresolver.go | 13 ++- xds/balancer/clusterresolver/config.go | 12 +-- xds/balancer/clusterresolver/configbuilder.go | 13 ++- xds/balancer/clusterresolver/logging.go | 13 ++- .../clusterresolver/resource_resolver.go | 13 ++- .../clusterresolver/resource_resolver_dns.go | 13 ++- .../clusterresolver/weightedtarget_config.go | 13 ++- xds/balancer/loadstore/load_store_wrapper.go | 13 ++- xds/balancer/orca/orca.go | 12 +-- xds/balancer/priority/balancer.go | 13 ++- xds/balancer/priority/balancer_child.go | 13 ++- xds/balancer/priority/balancer_priority.go | 13 ++- xds/balancer/priority/config.go | 13 ++- xds/balancer/priority/config_test.go | 13 ++- xds/balancer/priority/ignore_resolve_now.go | 13 ++- xds/balancer/priority/logging.go | 13 ++- xds/balancer/priority/utils.go | 13 ++- xds/balancer/priority/utils_test.go | 13 ++- xds/balancer/ringhash/config.go | 13 ++- xds/balancer/ringhash/config_test.go | 13 ++- xds/balancer/ringhash/logging.go | 13 ++- xds/balancer/ringhash/picker.go | 13 ++- xds/balancer/ringhash/ring.go | 13 ++- xds/balancer/ringhash/ring_test.go | 13 ++- xds/balancer/ringhash/ringhash.go | 13 ++- xds/balancer/ringhash/util.go | 13 ++- xds/client/attributes.go | 12 +-- xds/client/authority.go | 12 +-- xds/client/bootstrap/bootstrap.go | 13 ++- xds/client/bootstrap/bootstrap_test.go | 18 ++-- xds/client/bootstrap/logging.go | 13 ++- xds/client/bootstrap/template.go | 12 +-- xds/client/bootstrap/template_test.go | 12 +-- xds/client/client.go | 13 ++- xds/client/controller.go | 12 +-- xds/client/controller/controller.go | 12 +-- xds/client/controller/loadreport.go | 12 +-- xds/client/controller/transport.go | 17 ++-- xds/client/controller/version/v2/client.go | 13 ++- .../controller/version/v2/loadreport.go | 13 ++- xds/client/controller/version/v3/client.go | 13 ++- .../controller/version/v3/loadreport.go | 13 ++- xds/client/controller/version/version.go | 12 +-- xds/client/dump.go | 13 ++- xds/client/load/reporter.go | 13 ++- xds/client/load/store.go | 11 ++- xds/client/load/store_test.go | 12 +-- xds/client/loadreport.go | 12 +-- xds/client/logging.go | 13 ++- xds/client/pubsub/dump.go | 12 +-- xds/client/pubsub/interface.go | 12 +-- xds/client/pubsub/pubsub.go | 12 +-- xds/client/pubsub/update.go | 12 +-- xds/client/pubsub/watch.go | 12 +-- xds/client/requests_counter.go | 13 ++- xds/client/resource/errors.go | 13 ++- xds/client/resource/filter_chain.go | 14 +-- xds/client/resource/locality_id.go | 14 +-- xds/client/resource/matcher.go | 12 +-- xds/client/resource/matcher_path.go | 12 +-- xds/client/resource/name.go | 12 +-- xds/client/resource/type.go | 12 +-- xds/client/resource/type_cds.go | 12 +-- xds/client/resource/type_eds.go | 12 +-- xds/client/resource/type_lds.go | 12 +-- xds/client/resource/type_rds.go | 12 +-- xds/client/resource/unmarshal.go | 12 +-- xds/client/resource/unmarshal_cds.go | 12 +-- xds/client/resource/unmarshal_eds.go | 12 +-- xds/client/resource/unmarshal_lds.go | 18 ++-- xds/client/resource/unmarshal_rds.go | 14 +-- xds/client/resource/version/version.go | 13 ++- xds/client/singleton.go | 13 ++- xds/client/watchers.go | 12 +-- xds/clusterspecifier/cluster_specifier.go | 13 ++- xds/csds/csds.go | 13 ++- xds/httpfilter/fault/fault.go | 13 ++- xds/httpfilter/httpfilter.go | 13 ++- xds/httpfilter/rbac/rbac.go | 13 ++- xds/httpfilter/router/router.go | 13 ++- xds/internal/internal.go | 12 +-- xds/resolver/logging.go | 13 ++- xds/resolver/serviceconfig.go | 15 ++- xds/resolver/watch_service.go | 13 ++- xds/resolver/xds_resolver.go | 12 +-- xds/server/conn_wrapper.go | 13 ++- xds/server/listener_wrapper.go | 19 ++-- xds/server/rds_handler.go | 13 ++- xds/utils/backoff/backoff.go | 13 ++- xds/utils/balancer/stub/stub.go | 13 ++- xds/utils/balancergroup/balancergroup.go | 12 +-- .../balancergroup/balancerstateaggregator.go | 13 ++- xds/utils/balancerload/load.go | 12 +-- xds/utils/buffer/unbounded.go | 12 +-- xds/utils/credentials/xds/handshake_info.go | 13 ++- .../credentials/xds/handshake_info_test.go | 13 ++- xds/utils/envconfig/envconfig.go | 13 ++- xds/utils/envconfig/xds.go | 13 ++- xds/utils/grpclog/grpclog.go | 13 ++- xds/utils/grpclog/prefixLogger.go | 13 ++- xds/utils/grpcrand/grpcrand.go | 13 ++- xds/utils/grpcsync/event.go | 13 ++- xds/utils/grpcutil/encode_duration.go | 13 ++- xds/utils/grpcutil/encode_duration_test.go | 13 ++- xds/utils/grpcutil/grpcutil.go | 13 ++- xds/utils/grpcutil/metadata.go | 13 ++- xds/utils/grpcutil/method.go | 13 ++- xds/utils/grpcutil/method_test.go | 13 ++- xds/utils/grpcutil/regex.go | 13 ++- xds/utils/grpcutil/regex_test.go | 13 ++- xds/utils/hierarchy/hierarchy.go | 13 ++- xds/utils/hierarchy/hierarchy_test.go | 13 ++- xds/utils/matcher/matcher_header.go | 13 ++- xds/utils/matcher/matcher_header_test.go | 13 ++- xds/utils/matcher/regex.go | 13 ++- xds/utils/matcher/regex_test.go | 13 ++- xds/utils/matcher/string_matcher.go | 13 ++- xds/utils/matcher/string_matcher_test.go | 13 ++- xds/utils/metadata/metadata.go | 13 ++- xds/utils/pretty/pretty.go | 13 ++- xds/utils/rbac/matchers.go | 11 ++- xds/utils/rbac/rbac_engine.go | 11 ++- xds/utils/resolver/config_selector.go | 13 ++- xds/utils/resolver/passthrough/passthrough.go | 13 ++- xds/utils/resolver/unix/unix.go | 13 ++- xds/utils/serviceconfig/serviceconfig.go | 13 ++- xds/utils/serviceconfig/serviceconfig_test.go | 15 ++- xds/utils/transport/conn.go | 17 ++++ .../transport/networktype/networktype.go | 13 ++- xds/utils/wrr/edf.go | 12 +-- xds/utils/wrr/random.go | 12 +-- xds/utils/wrr/wrr.go | 12 +-- xds/utils/xds_cache/timeoutCache.go | 12 +-- xds/xds_handshake_cluster.go | 11 ++- 157 files changed, 1148 insertions(+), 992 deletions(-) create mode 100644 filter/xds/cb/filter.go diff --git a/common/constant/key.go b/common/constant/key.go index 1a1c8be520..59ec1fb884 100644 --- a/common/constant/key.go +++ b/common/constant/key.go @@ -93,6 +93,7 @@ const ( TokenFilterKey = "token" TpsLimitFilterKey = "tps" TracingFilterKey = "tracing" + XdsCircuitBreakerKey = "xds_circuit_reaker" ) const ( diff --git a/common/constant/xds.go b/common/constant/xds.go index 6a1297051f..53a0f8c2e0 100644 --- a/common/constant/xds.go +++ b/common/constant/xds.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package constant const ( diff --git a/common/url.go b/common/url.go index 0ac969ae19..ac146ed1a7 100644 --- a/common/url.go +++ b/common/url.go @@ -875,3 +875,21 @@ func (c *URL) GetParamDuration(s string, d string) time.Duration { } return 3 * time.Second } + +func GetSubscribeName(url *URL) string { + var buffer bytes.Buffer + + buffer.Write([]byte(DubboNodes[PROVIDER])) + appendParam(&buffer, url, constant.InterfaceKey) + appendParam(&buffer, url, constant.VersionKey) + appendParam(&buffer, url, constant.GroupKey) + return buffer.String() +} + +func appendParam(target *bytes.Buffer, url *URL, key string) { + value := url.GetParam(key, "") + target.Write([]byte(constant.NacosServiceNameSeparator)) + if strings.TrimSpace(value) != "" { + target.Write([]byte(value)) + } +} diff --git a/filter/xds/cb/filter.go b/filter/xds/cb/filter.go new file mode 100644 index 0000000000..7ddf947fcd --- /dev/null +++ b/filter/xds/cb/filter.go @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cb + +import ( + "context" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/extension" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/filter" + "dubbo.apache.org/dubbo-go/v3/protocol" + "dubbo.apache.org/dubbo-go/v3/remoting/xds" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// this should be executed before users set their own Tracer +func init() { + extension.SetFilter(constant.XdsCircuitBreakerKey, newCircuitBreakerFilter) +} + +// if you wish to using opentracing, please add the this filter into your filter attribute in your configure file. +// notice that this could be used in both client-side and server-side. +type circuitBreakerFilter struct { + client *xds.WrappedClient +} + +func (cb *circuitBreakerFilter) Invoke(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { + url := invoker.GetURL() + rejectedExeHandler := url.GetParam(constant.DefaultKey, constant.DefaultKey) + clusterUpdate, err := cb.getClusterUpdate(url) + if err != nil { + logger.Errorf("xds circuitBreakerFilter get request counter fail", err) + return nil + } + counter := client.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName) + if err := counter.StartRequest(*clusterUpdate.MaxRequests); err != nil { + rejectedExecutionHandler, err := extension.GetRejectedExecutionHandler(rejectedExeHandler) + if err != nil { + logger.Warn(err) + } else { + return rejectedExecutionHandler.RejectedExecution(url, invocation) + } + } + return invoker.Invoke(ctx, invocation) +} + +func (cb *circuitBreakerFilter) OnResponse(ctx context.Context, result protocol.Result, + invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { + url := invoker.GetURL() + clusterUpdate, err := cb.getClusterUpdate(url) + if err != nil { + logger.Errorf("xds circuitBreakerFilter get request counter fail", err) + return nil + } + counter := client.GetClusterRequestsCounter(clusterUpdate.ClusterName, clusterUpdate.EDSServiceName) + counter.EndRequest() + return result +} + +var circuitBreakerFilterInstance filter.Filter + +func newCircuitBreakerFilter() filter.Filter { + if circuitBreakerFilterInstance == nil { + circuitBreakerFilterInstance = &circuitBreakerFilter{ + client: xds.GetXDSWrappedClient(), + } + } + return circuitBreakerFilterInstance +} + +func (cb *circuitBreakerFilter) getClusterUpdate(url *common.URL) (resource.ClusterUpdate, error) { + hostAddr, err := cb.client.GetHostAddrByServiceUniqueKey(common.GetSubscribeName(url)) + if err != nil { + logger.Errorf("xds circuitBreakerFilter get GetHostAddrByServiceUniqueKey fail", err) + return resource.ClusterUpdate{}, err + } + clusterUpdate := cb.client.GetClusterUpdateIgnoreVersion(hostAddr) + return clusterUpdate, nil +} diff --git a/registry/event_test.go b/registry/event_test.go index cebecf9199..e772056b30 100644 --- a/registry/event_test.go +++ b/registry/event_test.go @@ -34,7 +34,7 @@ func TestKey(t *testing.T) { se := ServiceEvent{ Service: u1, } - assert.Equal(t, se.Key(), "dubbo://:@127.0.0.1:20000/?interface=com.ikurento.user.UserProvider&group=&version=2.0×tamp=") + assert.Equal(t, se.Key(), "dubbo://:@127.0.0.1:20000/?interface=com.ikurento.user.UserProvider&group=&version=2.0×tamp=&meshClusterId=") se2 := ServiceEvent{ Service: u1, diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 66f67c688f..c6b5f2fa3d 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package xds import ( @@ -5,7 +22,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "strings" "sync" "time" @@ -173,7 +189,7 @@ func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr Addr) (*W func (w *WrappedClient) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s:8080/debug/adsz", w.istiodPodIP), nil) - token, err := os.ReadFile(istiodTokenPath) + token, err := ioutil.ReadFile(istiodTokenPath) if err != nil { return nil, err } @@ -229,12 +245,25 @@ func getHostNameAndPortFromAddr(hostAddr string) (string, string) { return hostName, port } +func (w *WrappedClient) GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate { + hostName, port := getHostNameAndPortFromAddr(hostAddr) + w.cdsMapLock.RLock() + defer w.cdsMapLock.Unlock() + for clusterName, v := range w.cdsMap { + clusterNameData := strings.Split(clusterName, "|") + if clusterNameData[1] == port && clusterNameData[3] == hostName { + return v + } + } + return resource.ClusterUpdate{} +} + func (w *WrappedClient) getAllVersionClusterName(hostAddr string) []string { hostName, port := getHostNameAndPortFromAddr(hostAddr) allVersionClusterNames := make([]string, 0) w.cdsMapLock.RLock() defer w.cdsMapLock.Unlock() - for clusterName, _ := range w.cdsMap { + for clusterName := range w.cdsMap { clusterNameData := strings.Split(clusterName, "|") if clusterNameData[1] == port && clusterNameData[3] == hostName { allVersionClusterNames = append(allVersionClusterNames, clusterName) @@ -296,7 +325,7 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s w.hostAddrClusterCtxMapLock.Unlock() oldlisteningClusterMap := make(map[string]bool) - for cluster, _ := range listeningClustersCancelMap { + for cluster := range listeningClustersCancelMap { oldlisteningClusterMap[cluster] = false } for _, updatedClusterName := range updatedAllVersionedClusterName { @@ -485,7 +514,7 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! if clusterNameList[3] == w.istiodAddr.HostnameOrIP { // 1. find istiod podIP - // todo: When would eds level watch be cancelled? + // todo: When would eds level watch be canceled? cancel1 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundIstiod { return @@ -502,7 +531,7 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { return } // 2. found local hostAddr - // todo: When would eds level watch be cancelled? + // todo: When would eds level watch be canceled? cancel2 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundLocal { return @@ -580,7 +609,7 @@ func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { } func (w *WrappedClient) runWatchingResource() { - for _ = range w.cdsUpdateEventChan { + for range w.cdsUpdateEventChan { w.cdsUpdateEventHandlersLock.RLock() for _, h := range w.cdsUpdateEventHandlers { h() diff --git a/remoting/xds/debug.go b/remoting/xds/debug.go index 067e66ea9f..ed8560268c 100644 --- a/remoting/xds/debug.go +++ b/remoting/xds/debug.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package xds import ( diff --git a/remoting/xds/ewatcher.go b/remoting/xds/ewatcher.go index bd667c77e8..67b3d83438 100644 --- a/remoting/xds/ewatcher.go +++ b/remoting/xds/ewatcher.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package xds import ( diff --git a/remoting/xds/model.go b/remoting/xds/model.go index 7fb823b2ab..a053007d37 100644 --- a/remoting/xds/model.go +++ b/remoting/xds/model.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package xds import ( diff --git a/test/xds/main.go b/test/xds/main.go index cc9015b749..85b661f329 100644 --- a/test/xds/main.go +++ b/test/xds/main.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package main import ( diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go index 490124cab5..b3e02650e6 100644 --- a/xds/balancer/balancer.go +++ b/xds/balancer/balancer.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package balancer installs all the xds balancers. diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index ab95d5687f..f4cf31fa9e 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go index dec61af342..9e3d8c67ac 100644 --- a/xds/balancer/cdsbalancer/cluster_handler.go +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -1,9 +1,10 @@ /* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go index e19b4d0cad..547b4d9faa 100644 --- a/xds/balancer/cdsbalancer/logging.go +++ b/xds/balancer/cdsbalancer/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package cdsbalancer diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go index c79a866dd7..b93266fe57 100644 --- a/xds/balancer/clusterimpl/clusterimpl.go +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package clusterimpl implements the xds_cluster_impl balancing policy. It diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go index 9fe1a41a7e..59aba156db 100644 --- a/xds/balancer/clusterimpl/config.go +++ b/xds/balancer/clusterimpl/config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go index 0bdaa66cf9..baf0c18f03 100644 --- a/xds/balancer/clusterimpl/config_test.go +++ b/xds/balancer/clusterimpl/config_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go index b94aa44256..790a50676e 100644 --- a/xds/balancer/clusterimpl/logging.go +++ b/xds/balancer/clusterimpl/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go index 39874a6f86..183d852b59 100644 --- a/xds/balancer/clusterimpl/picker.go +++ b/xds/balancer/clusterimpl/picker.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterimpl diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go index 8b0ad174df..07eb81e56c 100644 --- a/xds/balancer/clustermanager/balancerstateaggregator.go +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clustermanager diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go index 16517579f8..3dfc10a6fd 100644 --- a/xds/balancer/clustermanager/clustermanager.go +++ b/xds/balancer/clustermanager/clustermanager.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package clustermanager implements the cluster manager LB policy for xds. diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go index 7657473966..9121a328f3 100644 --- a/xds/balancer/clustermanager/config.go +++ b/xds/balancer/clustermanager/config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clustermanager diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go index a8ead24c5b..fcec6d6b90 100644 --- a/xds/balancer/clustermanager/picker.go +++ b/xds/balancer/clustermanager/picker.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clustermanager diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go index a871cd36e9..0185907b67 100644 --- a/xds/balancer/clusterresolver/clusterresolver.go +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package clusterresolver contains EDS balancer implementation. diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go index dd33636459..a70d32d389 100644 --- a/xds/balancer/clusterresolver/config.go +++ b/xds/balancer/clusterresolver/config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go index f9a031a5eb..8ca82c8a98 100644 --- a/xds/balancer/clusterresolver/configbuilder.go +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go index 8809bcac92..e1a7d3e2f7 100644 --- a/xds/balancer/clusterresolver/logging.go +++ b/xds/balancer/clusterresolver/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go index 7ab8e75e38..766e327df0 100644 --- a/xds/balancer/clusterresolver/resource_resolver.go +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go index 2d3553d121..5315ae3830 100644 --- a/xds/balancer/clusterresolver/resource_resolver_dns.go +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go index 1c221d95b6..f3b5d4562c 100644 --- a/xds/balancer/clusterresolver/weightedtarget_config.go +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package clusterresolver diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go index 4da720fb74..8fb977a526 100644 --- a/xds/balancer/loadstore/load_store_wrapper.go +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package loadstore contains the loadStoreWrapper shared by the balancers. diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go index 0a635cebbd..7553269704 100644 --- a/xds/balancer/orca/orca.go +++ b/xds/balancer/orca/orca.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // Package orca implements Open Request Cost Aggregation. package orca diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go index 977078fe5f..19405dc180 100644 --- a/xds/balancer/priority/balancer.go +++ b/xds/balancer/priority/balancer.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package priority implements the priority balancer. diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go index 9bd4463300..fbaf38dc98 100644 --- a/xds/balancer/priority/balancer_child.go +++ b/xds/balancer/priority/balancer_child.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go index 36b823d41e..3e4166dec8 100644 --- a/xds/balancer/priority/balancer_priority.go +++ b/xds/balancer/priority/balancer_priority.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go index 0519465124..f9c236e321 100644 --- a/xds/balancer/priority/config.go +++ b/xds/balancer/priority/config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go index 1cee0c728b..2cdbd90f46 100644 --- a/xds/balancer/priority/config_test.go +++ b/xds/balancer/priority/config_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go index a4eaa36271..24c9e2db13 100644 --- a/xds/balancer/priority/ignore_resolve_now.go +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go index 2e7874c23b..392992969b 100644 --- a/xds/balancer/priority/logging.go +++ b/xds/balancer/priority/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/utils.go b/xds/balancer/priority/utils.go index 45fbe76443..7724d77391 100644 --- a/xds/balancer/priority/utils.go +++ b/xds/balancer/priority/utils.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go index 947532d92e..3bea5419b8 100644 --- a/xds/balancer/priority/utils_test.go +++ b/xds/balancer/priority/utils_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package priority diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go index 71625a0400..475e83ec26 100644 --- a/xds/balancer/ringhash/config.go +++ b/xds/balancer/ringhash/config.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go index ab703c8d8e..075623bf72 100644 --- a/xds/balancer/ringhash/config_test.go +++ b/xds/balancer/ringhash/config_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go index 2ed8ea8774..eb7457a46f 100644 --- a/xds/balancer/ringhash/logging.go +++ b/xds/balancer/ringhash/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go index c7bd9a8154..39314b9689 100644 --- a/xds/balancer/ringhash/picker.go +++ b/xds/balancer/ringhash/picker.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go index f0db1e6525..f1dba7d849 100644 --- a/xds/balancer/ringhash/ring.go +++ b/xds/balancer/ringhash/ring.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go index abf23bbf25..2dbb7b5610 100644 --- a/xds/balancer/ringhash/ring_test.go +++ b/xds/balancer/ringhash/ring_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go index 4001f7737b..1f5f3156b6 100644 --- a/xds/balancer/ringhash/ringhash.go +++ b/xds/balancer/ringhash/ringhash.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package ringhash implements the ringhash balancer. diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go index 05ad2c9555..3bf0ad81e4 100644 --- a/xds/balancer/ringhash/util.go +++ b/xds/balancer/ringhash/util.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package ringhash diff --git a/xds/client/attributes.go b/xds/client/attributes.go index 986750f2e8..9bad8501f8 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -1,9 +1,10 @@ /* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -12,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package client diff --git a/xds/client/authority.go b/xds/client/authority.go index 5be5d5b1eb..35659993e7 100644 --- a/xds/client/authority.go +++ b/xds/client/authority.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 3e87ba198f..e7d547a5e7 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package bootstrap provides the functionality to initialize certain aspects diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index e66a34e38f..551b135bf2 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package bootstrap @@ -44,7 +43,6 @@ import ( import ( "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" - internal2 "dubbo.apache.org/dubbo-go/v3/xds/internal" "dubbo.apache.org/dubbo-go/v3/xds/utils/envconfig" ) @@ -557,7 +555,7 @@ type fakeCertProvider struct { func TestNewConfigWithCertificateProviders(t *testing.T) { return - bootstrapFileMap := map[string]string{ + /*bootstrapFileMap := map[string]string{ "badJSONCertProviderConfig": ` { "node": { @@ -711,7 +709,7 @@ func TestNewConfigWithCertificateProviders(t *testing.T) { testNewConfigWithFileNameEnv(t, test.name, test.wantErr, test.wantConfig) testNewConfigWithFileContentEnv(t, test.name, test.wantErr, test.wantConfig) }) - } + }*/ } func TestNewConfigWithServerListenerResourceNameTemplate(t *testing.T) { diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go index 3ed8b8dcef..5ad007c96f 100644 --- a/xds/client/bootstrap/logging.go +++ b/xds/client/bootstrap/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package bootstrap diff --git a/xds/client/bootstrap/template.go b/xds/client/bootstrap/template.go index 9b51fcc839..58e8e9b708 100644 --- a/xds/client/bootstrap/template.go +++ b/xds/client/bootstrap/template.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go index 9751e79a43..ca3251d2ce 100644 --- a/xds/client/bootstrap/template_test.go +++ b/xds/client/bootstrap/template_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/client.go b/xds/client/client.go index 3d8e502f99..d65dc4f250 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // package client implements a full fledged gRPC client for the xDS API used diff --git a/xds/client/controller.go b/xds/client/controller.go index 9f53a3b738..12556d016d 100644 --- a/xds/client/controller.go +++ b/xds/client/controller.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go index 78743722e3..4bf2fa6104 100644 --- a/xds/client/controller/controller.go +++ b/xds/client/controller/controller.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go index 52d82fb320..3b9e55041f 100644 --- a/xds/client/controller/loadreport.go +++ b/xds/client/controller/loadreport.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go index 3953dfa20e..aafbda85ec 100644 --- a/xds/client/controller/transport.go +++ b/xds/client/controller/transport.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package controller @@ -374,7 +373,7 @@ func (t *Controller) processAckInfo(ack *ackAction, stream grpc.ClientStream) (t } // reportLoad starts an LRS stream to report load data to the management server. -// It blocks until the context is cancelled. +// It blocks until the context is canceled. func (t *Controller) reportLoad(ctx context.Context, cc *grpc.ClientConn, opts controllerversion.LoadReportingOptions) { retries := 0 for { @@ -402,7 +401,7 @@ func (t *Controller) reportLoad(ctx context.Context, cc *grpc.ClientConn, opts c } t.logger.Infof("lrs: created LRS stream") - if err := t.vClient.SendFirstLoadStatsRequest(stream); err != nil { + if err = t.vClient.SendFirstLoadStatsRequest(stream); err != nil { t.logger.Warningf("lrs: failed to send first request: %v", err) continue } diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go index d67e8b80ab..36c6d24818 100644 --- a/xds/client/controller/version/v2/client.go +++ b/xds/client/controller/version/v2/client.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package v2 provides xDS v2 transport protocol specific functionality. diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go index 60cd6b3a05..55763e236c 100644 --- a/xds/client/controller/version/v2/loadreport.go +++ b/xds/client/controller/version/v2/loadreport.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package v2 diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go index ec15a7b711..a7bcf505a4 100644 --- a/xds/client/controller/version/v3/client.go +++ b/xds/client/controller/version/v3/client.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package v3 provides xDS v3 transport protocol specific functionality. diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go index 800b10d2c1..5a778eac6d 100644 --- a/xds/client/controller/version/v3/loadreport.go +++ b/xds/client/controller/version/v3/loadreport.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package v3 diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go index 439b9bb93c..d45ec17ca2 100644 --- a/xds/client/controller/version/version.go +++ b/xds/client/controller/version/version.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/dump.go b/xds/client/dump.go index f01178b0ae..51d386623a 100644 --- a/xds/client/dump.go +++ b/xds/client/dump.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package client diff --git a/xds/client/load/reporter.go b/xds/client/load/reporter.go index 67e29e5bae..a0ba224f24 100644 --- a/xds/client/load/reporter.go +++ b/xds/client/load/reporter.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package load diff --git a/xds/client/load/store.go b/xds/client/load/store.go index 551a5147b6..14fc5308f5 100644 --- a/xds/client/load/store.go +++ b/xds/client/load/store.go @@ -1,9 +1,10 @@ /* - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go index 9cb641a4c0..be4b77e7b4 100644 --- a/xds/client/load/store_test.go +++ b/xds/client/load/store_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/loadreport.go b/xds/client/loadreport.go index bb45e013dc..97695038bc 100644 --- a/xds/client/loadreport.go +++ b/xds/client/loadreport.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/logging.go b/xds/client/logging.go index 172d5858aa..6949dd77f3 100644 --- a/xds/client/logging.go +++ b/xds/client/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package client diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go index 490bde0709..d4895cd4c0 100644 --- a/xds/client/pubsub/dump.go +++ b/xds/client/pubsub/dump.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go index 5875a59525..be1be057a8 100644 --- a/xds/client/pubsub/interface.go +++ b/xds/client/pubsub/interface.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go index c14a1dcbbe..32385118d7 100644 --- a/xds/client/pubsub/pubsub.go +++ b/xds/client/pubsub/pubsub.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 113167c540..1f0787426d 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go index 120e2231a5..6779c4fdd9 100644 --- a/xds/client/pubsub/watch.go +++ b/xds/client/pubsub/watch.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/requests_counter.go b/xds/client/requests_counter.go index 04ee728dc8..0d93d10327 100644 --- a/xds/client/requests_counter.go +++ b/xds/client/requests_counter.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package client diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go index 23643bf1e4..82b51520da 100644 --- a/xds/client/resource/errors.go +++ b/xds/client/resource/errors.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package resource diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go index c270aba0fa..50342ecb9e 100644 --- a/xds/client/resource/filter_chain.go +++ b/xds/client/resource/filter_chain.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -525,7 +525,7 @@ func (fci *FilterChainManager) filterChainFromProto(fc *v3listenerpb.FilterChain return nil, fmt.Errorf("transport_socket field has unexpected typeURL: %s", any.TypeUrl) } downstreamCtx := &v3tlspb.DownstreamTlsContext{} - if err := proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil { + if err = proto.Unmarshal(any.GetValue(), downstreamCtx); err != nil { return nil, fmt.Errorf("failed to unmarshal DownstreamTlsContext in LDS response: %v", err) } if downstreamCtx.GetRequireSni().GetValue() { diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go index 56eff7b437..e4ecd4a93c 100644 --- a/xds/client/resource/locality_id.go +++ b/xds/client/resource/locality_id.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -38,7 +38,7 @@ type LocalityID struct { SubZone string `json:"subZone,omitempty"` } -// ToString generates a string representation of LocalityID by marshalling it into +// ToString generates a string representation of LocalityID by marshaling it into // json. Not calling it String() so printf won't call it. func (l LocalityID) ToString() (string, error) { b, err := json.Marshal(l) diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go index 93ccda43f1..0605634aa0 100644 --- a/xds/client/resource/matcher.go +++ b/xds/client/resource/matcher.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go index 9310d3b7bf..f2565e97fe 100644 --- a/xds/client/resource/matcher_path.go +++ b/xds/client/resource/matcher_path.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go index 48a636e62a..1aaaad31a2 100644 --- a/xds/client/resource/name.go +++ b/xds/client/resource/name.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go index 7dff2dd430..c4be177987 100644 --- a/xds/client/resource/type.go +++ b/xds/client/resource/type.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go index 4d101817aa..16bef47d5f 100644 --- a/xds/client/resource/type_cds.go +++ b/xds/client/resource/type_cds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/type_eds.go b/xds/client/resource/type_eds.go index 30627a6805..88b5c09594 100644 --- a/xds/client/resource/type_eds.go +++ b/xds/client/resource/type_eds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go index 83e63505d5..cd5403b0fc 100644 --- a/xds/client/resource/type_lds.go +++ b/xds/client/resource/type_lds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go index 9b9b7ca806..999d9e6c68 100644 --- a/xds/client/resource/type_rds.go +++ b/xds/client/resource/type_rds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go index 29c09bbe3e..c02d3d7b30 100644 --- a/xds/client/resource/unmarshal.go +++ b/xds/client/resource/unmarshal.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go index ef6e753532..8af5af0650 100644 --- a/xds/client/resource/unmarshal_cds.go +++ b/xds/client/resource/unmarshal_cds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go index 79a18f65a0..3d0bfdf29f 100644 --- a/xds/client/resource/unmarshal_eds.go +++ b/xds/client/resource/unmarshal_eds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index 06d1d035a3..ddda349c5a 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -153,14 +153,14 @@ func unwrapHTTPFilterConfig(config *anypb.Any) (proto.Message, string, error) { // The real type name is inside the new TypedStruct message. s := new(v3cncftypepb.TypedStruct) if err := ptypes.UnmarshalAny(config, s); err != nil { - return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + return nil, "", fmt.Errorf("error unmarshaling TypedStruct filter config: %v", err) } return s, s.GetTypeUrl(), nil case ptypes.Is(config, &v1udpatypepb.TypedStruct{}): // The real type name is inside the old TypedStruct message. s := new(v1udpatypepb.TypedStruct) if err := ptypes.UnmarshalAny(config, s); err != nil { - return nil, "", fmt.Errorf("error unmarshalling TypedStruct filter config: %v", err) + return nil, "", fmt.Errorf("error unmarshaling TypedStruct filter config: %v", err) } return s, s.GetTypeUrl(), nil default: @@ -201,7 +201,7 @@ func processHTTPFilterOverrides(cfgs map[string]*anypb.Any) (map[string]httpfilt s := new(v3routepb.FilterConfig) if ptypes.Is(cfg, s) { if err := ptypes.UnmarshalAny(cfg, s); err != nil { - return nil, fmt.Errorf("filter override %q: error unmarshalling FilterConfig: %v", name, err) + return nil, fmt.Errorf("filter override %q: error unmarshaling FilterConfig: %v", name, err) } cfg = s.GetConfig() optional = s.GetIsOptional() diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go index 9868bc46b4..af44b32cab 100644 --- a/xds/client/resource/unmarshal_rds.go +++ b/xds/client/resource/unmarshal_rds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -177,7 +177,7 @@ func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) { cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)} for _, s := range strings.Split(rp.GetRetryOn(), ",") { switch strings.TrimSpace(strings.ToLower(s)) { - case "cancelled": + case "canceled": cfg.RetryOn[codes.Canceled] = true case "deadline-exceeded": cfg.RetryOn[codes.DeadlineExceeded] = true diff --git a/xds/client/resource/version/version.go b/xds/client/resource/version/version.go index edfa68762f..6fb1b3a22a 100644 --- a/xds/client/resource/version/version.go +++ b/xds/client/resource/version/version.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package version defines constants to distinguish between supported xDS API diff --git a/xds/client/singleton.go b/xds/client/singleton.go index 66daecfbec..c42877e00e 100644 --- a/xds/client/singleton.go +++ b/xds/client/singleton.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package client diff --git a/xds/client/watchers.go b/xds/client/watchers.go index 57f6506c73..acae4d1b2c 100644 --- a/xds/client/watchers.go +++ b/xds/client/watchers.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/clusterspecifier/cluster_specifier.go b/xds/clusterspecifier/cluster_specifier.go index 54776f20cf..43a994bc4b 100644 --- a/xds/clusterspecifier/cluster_specifier.go +++ b/xds/clusterspecifier/cluster_specifier.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package clusterspecifier contains the ClusterSpecifier interface and a registry for diff --git a/xds/csds/csds.go b/xds/csds/csds.go index 90435de88a..0182584d80 100644 --- a/xds/csds/csds.go +++ b/xds/csds/csds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package csds implements features to dump the status (xDS responses) the diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go index 90352b4209..a935e87226 100644 --- a/xds/httpfilter/fault/fault.go +++ b/xds/httpfilter/fault/fault.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package fault implements the Envoy Fault Injection HTTP filter. diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go index 7302f83921..d022ff3728 100644 --- a/xds/httpfilter/httpfilter.go +++ b/xds/httpfilter/httpfilter.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package httpfilter contains the HTTPFilter interface and a registry for diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go index a54e572103..d022b86244 100644 --- a/xds/httpfilter/rbac/rbac.go +++ b/xds/httpfilter/rbac/rbac.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package rbac implements the Envoy RBAC HTTP filter. diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go index 6a48af6dd2..3fbdec00d3 100644 --- a/xds/httpfilter/router/router.go +++ b/xds/httpfilter/router/router.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package router implements the Envoy Router HTTP filter. diff --git a/xds/internal/internal.go b/xds/internal/internal.go index b778deb60f..2ba3d47c84 100644 --- a/xds/internal/internal.go +++ b/xds/internal/internal.go @@ -1,9 +1,10 @@ /* - * Copyright 2016 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -12,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package internal contains gRPC-internal code, to avoid polluting diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go index 7a3b9bc081..04544ee10a 100644 --- a/xds/resolver/logging.go +++ b/xds/resolver/logging.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package resolver diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go index 5122bf444b..ef44f757ee 100644 --- a/xds/resolver/serviceconfig.go +++ b/xds/resolver/serviceconfig.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package resolver @@ -261,7 +260,7 @@ func (cs *configSelector) generateHash(rpcInfo iresolver.RPCInfo, hashPolicies [ } // Deterministically combine the hash policies. Rotating prevents - // duplicate hash policies from cancelling each other out and preserves + // duplicate hash policies from canceling each other out and preserves // the 64 bits of entropy. if generatedPolicyHash { hash = bits.RotateLeft64(hash, 1) diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go index acb27b10d8..3f9e4f98a4 100644 --- a/xds/resolver/watch_service.go +++ b/xds/resolver/watch_service.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package resolver diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go index f444d82e82..436eabd5c8 100644 --- a/xds/resolver/xds_resolver.go +++ b/xds/resolver/xds_resolver.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -12,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package resolver implements the xds resolver, that does LDS and RDS to find diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index 47fc8e7d58..d3bead7b52 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package server diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go index 3a7ded1fc1..fe474eeb49 100644 --- a/xds/server/listener_wrapper.go +++ b/xds/server/listener_wrapper.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package server contains internal server-side functionality used by the public @@ -136,7 +135,7 @@ func NewListenerWrapper(params ListenerWrapperParams) (net.Listener, <-chan stru lw.logger.Infof("Watch started on resource name %v", lw.name) lw.cancelWatch = func() { cancelWatch() - lw.logger.Infof("Watch cancelled on resource name %v", lw.name) + lw.logger.Infof("Watch canceled on resource name %v", lw.name) } go lw.run() return lw, lw.goodUpdate.Done() @@ -174,10 +173,10 @@ type listenerWrapper struct { // keep track of whether the received update is the first one or not. goodUpdate *grpcsync.Event // A small race exists in the XDSClient code between the receipt of an xDS - // response and the user cancelling the associated watch. In this window, + // response and the user canceling the associated watch. In this window, // the registered callback may be invoked after the watch is canceled, and // the user is expected to work around this. This event signifies that the - // listener is closed (and hence the watch is cancelled), and we drop any + // listener is closed (and hence the watch is canceled), and we drop any // updates received in the callback if this event has fired. closed *grpcsync.Event diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go index 96b891cfed..6e6c8df8ef 100644 --- a/xds/server/rds_handler.go +++ b/xds/server/rds_handler.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package server diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go index 9101eb9fe1..538016d1b3 100644 --- a/xds/utils/backoff/backoff.go +++ b/xds/utils/backoff/backoff.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package backoff implement the backoff strategy for gRPC. diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go index 1a66fa9293..993f4c18bb 100644 --- a/xds/utils/balancer/stub/stub.go +++ b/xds/utils/balancer/stub/stub.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package stub implements a balancer for testing purposes. diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index 1f27a12b52..3c98c73547 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // Package balancergroup implements a utility struct to bind multiple balancers // into one balancer. package balancergroup diff --git a/xds/utils/balancergroup/balancerstateaggregator.go b/xds/utils/balancergroup/balancerstateaggregator.go index 1163943850..a7eed1144a 100644 --- a/xds/utils/balancergroup/balancerstateaggregator.go +++ b/xds/utils/balancergroup/balancerstateaggregator.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package balancergroup diff --git a/xds/utils/balancerload/load.go b/xds/utils/balancerload/load.go index 3a905d9665..e8c872d01f 100644 --- a/xds/utils/balancerload/load.go +++ b/xds/utils/balancerload/load.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - // Package balancerload defines APIs to parse server loads in trailers. The // parsed loads are sent to balancers in DoneInfo. package balancerload diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go index f056ab13ce..5049fb0716 100644 --- a/xds/utils/buffer/unbounded.go +++ b/xds/utils/buffer/unbounded.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -12,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package buffer provides an implementation of an unbounded buffer. diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index f51cf05069..b84b9f9b24 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package xds contains non-user facing functionality of the xds credentials. diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go index 55862c6968..5985e8464a 100644 --- a/xds/utils/credentials/xds/handshake_info_test.go +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package xds diff --git a/xds/utils/envconfig/envconfig.go b/xds/utils/envconfig/envconfig.go index 6f02725431..847348aae4 100644 --- a/xds/utils/envconfig/envconfig.go +++ b/xds/utils/envconfig/envconfig.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package envconfig contains grpc settings configured by environment variables. diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index de891fab5c..d0b5b83017 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package envconfig diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index 20eb1bcc8d..76d08cfb81 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package grpclog (internal) defines depth logging for grpc. diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go index 0083dbf0e7..5e277cec41 100644 --- a/xds/utils/grpclog/prefixLogger.go +++ b/xds/utils/grpclog/prefixLogger.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpclog diff --git a/xds/utils/grpcrand/grpcrand.go b/xds/utils/grpcrand/grpcrand.go index 740f83c2b7..8682bc3101 100644 --- a/xds/utils/grpcrand/grpcrand.go +++ b/xds/utils/grpcrand/grpcrand.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package grpcrand implements math/rand functions in a concurrent-safe way diff --git a/xds/utils/grpcsync/event.go b/xds/utils/grpcsync/event.go index fbe697c376..9a19a0e758 100644 --- a/xds/utils/grpcsync/event.go +++ b/xds/utils/grpcsync/event.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package grpcsync implements additional synchronization primitives built upon diff --git a/xds/utils/grpcutil/encode_duration.go b/xds/utils/grpcutil/encode_duration.go index b25b0baec3..b7242e9a20 100644 --- a/xds/utils/grpcutil/encode_duration.go +++ b/xds/utils/grpcutil/encode_duration.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/encode_duration_test.go b/xds/utils/grpcutil/encode_duration_test.go index eea49e2e77..d24e995855 100644 --- a/xds/utils/grpcutil/encode_duration_test.go +++ b/xds/utils/grpcutil/encode_duration_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/grpcutil.go b/xds/utils/grpcutil/grpcutil.go index e2f948e8f4..432a3cfee5 100644 --- a/xds/utils/grpcutil/grpcutil.go +++ b/xds/utils/grpcutil/grpcutil.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package grpcutil provides utility functions used across the gRPC codebase. diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go index 3d0ef900d8..d39bcfda1e 100644 --- a/xds/utils/grpcutil/metadata.go +++ b/xds/utils/grpcutil/metadata.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/method.go b/xds/utils/grpcutil/method.go index 4e7475060c..00b587090e 100644 --- a/xds/utils/grpcutil/method.go +++ b/xds/utils/grpcutil/method.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/method_test.go b/xds/utils/grpcutil/method_test.go index 36c786cffb..32491b4990 100644 --- a/xds/utils/grpcutil/method_test.go +++ b/xds/utils/grpcutil/method_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go index 410a4fe89e..35dd4e84bc 100644 --- a/xds/utils/grpcutil/regex.go +++ b/xds/utils/grpcutil/regex.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/grpcutil/regex_test.go b/xds/utils/grpcutil/regex_test.go index 4c12804fed..6fc0c95466 100644 --- a/xds/utils/grpcutil/regex_test.go +++ b/xds/utils/grpcutil/regex_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package grpcutil diff --git a/xds/utils/hierarchy/hierarchy.go b/xds/utils/hierarchy/hierarchy.go index 341d3405dc..e31a267292 100644 --- a/xds/utils/hierarchy/hierarchy.go +++ b/xds/utils/hierarchy/hierarchy.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package hierarchy contains functions to set and get hierarchy string from diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go index c17953f821..5a8a317dbb 100644 --- a/xds/utils/hierarchy/hierarchy_test.go +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package hierarchy diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go index ead9a67bfd..795001a36c 100644 --- a/xds/utils/matcher/matcher_header.go +++ b/xds/utils/matcher/matcher_header.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package matcher diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go index f8de56fafb..d14cf9f44d 100644 --- a/xds/utils/matcher/matcher_header_test.go +++ b/xds/utils/matcher/matcher_header_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package matcher diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go index 52c022f731..fef02c7058 100644 --- a/xds/utils/matcher/regex.go +++ b/xds/utils/matcher/regex.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package matcher diff --git a/xds/utils/matcher/regex_test.go b/xds/utils/matcher/regex_test.go index b26565336e..4c13b0a611 100644 --- a/xds/utils/matcher/regex_test.go +++ b/xds/utils/matcher/regex_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package matcher diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go index 632064b8d9..ab983846f0 100644 --- a/xds/utils/matcher/string_matcher.go +++ b/xds/utils/matcher/string_matcher.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package matcher contains types that need to be shared between code under diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go index d8d4dcfedd..4ff5cba93a 100644 --- a/xds/utils/matcher/string_matcher_test.go +++ b/xds/utils/matcher/string_matcher_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package matcher diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go index 232d92e588..7c5b83d3f6 100644 --- a/xds/utils/metadata/metadata.go +++ b/xds/utils/metadata/metadata.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package metadata diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go index e51a107388..12384f479d 100644 --- a/xds/utils/pretty/pretty.go +++ b/xds/utils/pretty/pretty.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package pretty defines helper functions to pretty-print structs for logging. diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go index 7aed8f6877..614bb689b8 100644 --- a/xds/utils/rbac/matchers.go +++ b/xds/utils/rbac/matchers.go @@ -1,9 +1,10 @@ /* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go index ae3f73b1a4..c782c4b4d0 100644 --- a/xds/utils/rbac/rbac_engine.go +++ b/xds/utils/rbac/rbac_engine.go @@ -1,9 +1,10 @@ /* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go index 5ac81255a3..a5d3a0413e 100644 --- a/xds/utils/resolver/config_selector.go +++ b/xds/utils/resolver/config_selector.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package resolver provides internal resolver-related functionality. diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go index 56db8e9ff3..93b3c52c2d 100644 --- a/xds/utils/resolver/passthrough/passthrough.go +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2017 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package passthrough implements a pass-through resolver. It sends the target diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go index d5e3af325a..45c4f6527a 100644 --- a/xds/utils/resolver/unix/unix.go +++ b/xds/utils/resolver/unix/unix.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package unix implements a resolver for unix targets. diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go index f9b6adcc67..9de85e9c5f 100644 --- a/xds/utils/serviceconfig/serviceconfig.go +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package serviceconfig contains utility functions to parse service config. diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go index 6936bedfce..fe8de63cc8 100644 --- a/xds/utils/serviceconfig/serviceconfig_test.go +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ package serviceconfig @@ -158,7 +157,7 @@ func TestBalancerConfigMarshalJSON(t *testing.T) { name: "OK config is nil", bc: BalancerConfig{ Name: testBalancerBuilderNotParserName, - Config: nil, // nil should be marshalled to an empty config "{}". + Config: nil, // nil should be marshaled to an empty config "{}". }, wantJSON: `[{"test-bb-not-parser": {}}]`, }, diff --git a/xds/utils/transport/conn.go b/xds/utils/transport/conn.go index 16caf9dd26..80728cb517 100644 --- a/xds/utils/transport/conn.go +++ b/xds/utils/transport/conn.go @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package transport import ( diff --git a/xds/utils/transport/networktype/networktype.go b/xds/utils/transport/networktype/networktype.go index c11b527827..b3686557fe 100644 --- a/xds/utils/transport/networktype/networktype.go +++ b/xds/utils/transport/networktype/networktype.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2020 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +13,6 @@ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. - * */ // Package networktype declares the network type to be used in the default diff --git a/xds/utils/wrr/edf.go b/xds/utils/wrr/edf.go index b4fb3f9d3b..382192f923 100644 --- a/xds/utils/wrr/edf.go +++ b/xds/utils/wrr/edf.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go index 459bf91040..cd591464f9 100644 --- a/xds/utils/wrr/random.go +++ b/xds/utils/wrr/random.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/utils/wrr/wrr.go b/xds/utils/wrr/wrr.go index d46bfad86e..86402960e1 100644 --- a/xds/utils/wrr/wrr.go +++ b/xds/utils/wrr/wrr.go @@ -1,10 +1,10 @@ /* - * - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * diff --git a/xds/utils/xds_cache/timeoutCache.go b/xds/utils/xds_cache/timeoutCache.go index d1c0c293a0..56098da23f 100644 --- a/xds/utils/xds_cache/timeoutCache.go +++ b/xds/utils/xds_cache/timeoutCache.go @@ -1,9 +1,10 @@ /* - * Copyright 2019 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * @@ -13,7 +14,6 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package xds_cache import ( diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index 0ba92b0ac7..cc38b63769 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -1,9 +1,10 @@ /* - * Copyright 2021 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * From 227b4ab24fdaed704cbf43417f68119fd7f9a2f6 Mon Sep 17 00:00:00 2001 From: Laurence <45508533+LaurenceLiZhixin@users.noreply.github.com> Date: Thu, 31 Mar 2022 22:28:09 +0800 Subject: [PATCH 13/19] Enhancement XDS ut. (#7) * fix: ut * Fix: add license --- .github/workflows/github-actions.yml | 2 +- .github/workflows/golangci-lint.yml | 4 +- .../{uniform_route.go => meshrouter.go} | 12 +- filter/xds/cb/filter.go | 2 +- go.mod | 115 ++- go.sum | 6 - integrate_test.sh | 2 + registry/mocks/NotifyListener.go | 43 + registry/xds/registry.go | 13 +- remoting/listener.go | 2 +- remoting/xds/client.go | 471 +++++------ remoting/xds/client_test.go | 764 ++++++++++++++++++ remoting/xds/{ => common}/model.go | 2 +- remoting/xds/error.go | 27 + remoting/xds/ewatcher.go | 66 -- remoting/xds/ewatcher/ewatcher.go | 131 +++ remoting/xds/ewatcher/ewatcher_test.go | 193 +++++ remoting/xds/ewatcher/mocks/EWatcher.go | 48 ++ remoting/xds/{ => interfaceMapping}/debug.go | 3 +- remoting/xds/interfaceMapping/debug_test.go | 52 ++ remoting/xds/interfaceMapping/handler.go | 156 ++++ remoting/xds/interfaceMapping/handler_test.go | 129 +++ remoting/xds/interfaceMapping/metadata.go | 49 ++ .../mocks/InterfaceMapHandler.go | 78 ++ remoting/xds/xds_client_factory.go | 62 ++ xds/balancer/balancer.go | 4 + xds/balancer/cdsbalancer/cdsbalancer.go | 4 + xds/balancer/cdsbalancer/cluster_handler.go | 4 + xds/balancer/cdsbalancer/logging.go | 4 + xds/balancer/clusterimpl/clusterimpl.go | 4 + xds/balancer/clusterimpl/config.go | 4 + xds/balancer/clusterimpl/config_test.go | 4 + xds/balancer/clusterimpl/logging.go | 4 + xds/balancer/clusterimpl/picker.go | 4 + .../clustermanager/balancerstateaggregator.go | 4 + xds/balancer/clustermanager/clustermanager.go | 4 + xds/balancer/clustermanager/config.go | 4 + xds/balancer/clustermanager/picker.go | 4 + .../clusterresolver/clusterresolver.go | 4 + xds/balancer/clusterresolver/config.go | 4 + xds/balancer/clusterresolver/configbuilder.go | 4 + xds/balancer/clusterresolver/logging.go | 4 + .../clusterresolver/resource_resolver.go | 4 + .../clusterresolver/resource_resolver_dns.go | 4 + .../clusterresolver/weightedtarget_config.go | 4 + xds/balancer/loadstore/load_store_wrapper.go | 4 + xds/balancer/orca/orca.go | 4 + xds/balancer/priority/balancer.go | 4 + xds/balancer/priority/balancer_child.go | 4 + xds/balancer/priority/balancer_priority.go | 4 + xds/balancer/priority/config.go | 4 + xds/balancer/priority/config_test.go | 4 + xds/balancer/priority/ignore_resolve_now.go | 4 + xds/balancer/priority/logging.go | 4 + xds/balancer/priority/utils.go | 4 + xds/balancer/priority/utils_test.go | 4 + xds/balancer/ringhash/config.go | 4 + xds/balancer/ringhash/config_test.go | 4 + xds/balancer/ringhash/logging.go | 4 + xds/balancer/ringhash/picker.go | 4 + xds/balancer/ringhash/ring.go | 4 + xds/balancer/ringhash/ring_test.go | 4 + xds/balancer/ringhash/ringhash.go | 4 + xds/balancer/ringhash/util.go | 4 + xds/client/attributes.go | 4 + xds/client/authority.go | 4 + xds/client/bootstrap/bootstrap.go | 4 + xds/client/bootstrap/bootstrap_test.go | 4 + xds/client/bootstrap/logging.go | 4 + xds/client/bootstrap/template.go | 4 + xds/client/bootstrap/template_test.go | 4 + xds/client/client.go | 4 + xds/client/controller.go | 4 + xds/client/controller/controller.go | 4 + xds/client/controller/loadreport.go | 4 + xds/client/controller/transport.go | 4 + xds/client/controller/version/v2/client.go | 4 + .../controller/version/v2/loadreport.go | 4 + xds/client/controller/version/v3/client.go | 4 + .../controller/version/v3/loadreport.go | 4 + xds/client/controller/version/version.go | 4 + xds/client/dump.go | 4 + xds/client/load/reporter.go | 4 + xds/client/load/store.go | 4 + xds/client/load/store_test.go | 4 + xds/client/loadreport.go | 4 + xds/client/logging.go | 4 + xds/client/mocks/XDSClient.go | 225 ++++++ xds/client/pubsub/dump.go | 4 + xds/client/pubsub/interface.go | 4 + xds/client/pubsub/pubsub.go | 4 + xds/client/pubsub/update.go | 4 + xds/client/pubsub/watch.go | 4 + xds/client/requests_counter.go | 4 + xds/client/resource/errors.go | 4 + xds/client/resource/filter_chain.go | 4 + xds/client/resource/locality_id.go | 4 + xds/client/resource/matcher.go | 4 + xds/client/resource/matcher_path.go | 4 + xds/client/resource/name.go | 4 + xds/client/resource/type.go | 4 + xds/client/resource/type_cds.go | 4 + xds/client/resource/type_eds.go | 4 + xds/client/resource/type_lds.go | 4 + xds/client/resource/type_rds.go | 4 + xds/client/resource/unmarshal.go | 4 + xds/client/resource/unmarshal_cds.go | 4 + xds/client/resource/unmarshal_eds.go | 4 + xds/client/resource/unmarshal_lds.go | 4 + xds/client/resource/unmarshal_rds.go | 7 +- xds/client/resource/version/version.go | 4 + xds/client/singleton.go | 4 + xds/client/watchers.go | 4 + xds/clusterspecifier/cluster_specifier.go | 4 + xds/csds/csds.go | 4 + xds/httpfilter/fault/fault.go | 4 + xds/httpfilter/httpfilter.go | 4 + xds/httpfilter/rbac/rbac.go | 4 + xds/httpfilter/router/router.go | 4 + xds/internal/internal.go | 4 + xds/resolver/logging.go | 4 + xds/resolver/serviceconfig.go | 4 + xds/resolver/watch_service.go | 4 + xds/resolver/xds_resolver.go | 4 + xds/server/conn_wrapper.go | 4 + xds/server/listener_wrapper.go | 4 + xds/server/rds_handler.go | 4 + xds/utils/backoff/backoff.go | 4 + xds/utils/balancer/stub/stub.go | 4 + xds/utils/balancergroup/balancergroup.go | 4 + .../balancergroup/balancerstateaggregator.go | 4 + xds/utils/balancerload/load.go | 4 + xds/utils/buffer/unbounded.go | 4 + xds/utils/credentials/xds/handshake_info.go | 4 + .../credentials/xds/handshake_info_test.go | 4 + xds/utils/envconfig/envconfig.go | 4 + xds/utils/envconfig/xds.go | 4 + xds/utils/grpclog/grpclog.go | 4 + xds/utils/grpclog/prefixLogger.go | 4 + xds/utils/grpcrand/grpcrand.go | 4 + xds/utils/grpcsync/event.go | 4 + xds/utils/grpcutil/encode_duration.go | 4 + xds/utils/grpcutil/encode_duration_test.go | 4 + xds/utils/grpcutil/grpcutil.go | 4 + xds/utils/grpcutil/metadata.go | 4 + xds/utils/grpcutil/method.go | 4 + xds/utils/grpcutil/method_test.go | 4 + xds/utils/grpcutil/regex.go | 4 + xds/utils/grpcutil/regex_test.go | 4 + xds/utils/hierarchy/hierarchy.go | 4 + xds/utils/hierarchy/hierarchy_test.go | 4 + xds/utils/matcher/matcher_header.go | 4 + xds/utils/matcher/matcher_header_test.go | 4 + xds/utils/matcher/regex.go | 4 + xds/utils/matcher/regex_test.go | 4 + xds/utils/matcher/string_matcher.go | 4 + xds/utils/matcher/string_matcher_test.go | 4 + xds/utils/metadata/metadata.go | 4 + xds/utils/pretty/pretty.go | 4 + xds/utils/rbac/matchers.go | 4 + xds/utils/rbac/rbac_engine.go | 4 + xds/utils/resolver/config_selector.go | 4 + xds/utils/resolver/passthrough/passthrough.go | 4 + xds/utils/resolver/unix/unix.go | 4 + xds/utils/serviceconfig/serviceconfig.go | 4 + xds/utils/serviceconfig/serviceconfig_test.go | 4 + xds/utils/transport/conn.go | 4 + .../transport/networktype/networktype.go | 4 + xds/utils/wrr/edf.go | 4 + xds/utils/wrr/random.go | 4 + xds/utils/wrr/wrr.go | 4 + xds/utils/xds_cache/timeoutCache.go | 4 + xds/xds_handshake_cluster.go | 4 + 173 files changed, 2863 insertions(+), 385 deletions(-) rename cluster/router/meshrouter/{uniform_route.go => meshrouter.go} (96%) create mode 100644 registry/mocks/NotifyListener.go create mode 100644 remoting/xds/client_test.go rename remoting/xds/{ => common}/model.go (98%) create mode 100644 remoting/xds/error.go delete mode 100644 remoting/xds/ewatcher.go create mode 100644 remoting/xds/ewatcher/ewatcher.go create mode 100644 remoting/xds/ewatcher/ewatcher_test.go create mode 100644 remoting/xds/ewatcher/mocks/EWatcher.go rename remoting/xds/{ => interfaceMapping}/debug.go (96%) create mode 100644 remoting/xds/interfaceMapping/debug_test.go create mode 100644 remoting/xds/interfaceMapping/handler.go create mode 100644 remoting/xds/interfaceMapping/handler_test.go create mode 100644 remoting/xds/interfaceMapping/metadata.go create mode 100644 remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go create mode 100644 remoting/xds/xds_client_factory.go create mode 100644 xds/client/mocks/XDSClient.go diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index 8fb6b810cc..3c6e5435ab 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -14,7 +14,7 @@ jobs: # If you want to matrix build , you can append the following list. matrix: go_version: - - 1.15 + - 1.17 os: - ubuntu-latest diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 918325c4e7..4c47124a12 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -12,11 +12,11 @@ jobs: strategy: matrix: golang: - - 1.15 + - 1.17 steps: - uses: actions/setup-go@v2 with: - go-version: 1.15 + go-version: 1.17 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3.1.0 diff --git a/cluster/router/meshrouter/uniform_route.go b/cluster/router/meshrouter/meshrouter.go similarity index 96% rename from cluster/router/meshrouter/uniform_route.go rename to cluster/router/meshrouter/meshrouter.go index 4be9c740d1..44c3c08ce0 100644 --- a/cluster/router/meshrouter/uniform_route.go +++ b/cluster/router/meshrouter/meshrouter.go @@ -23,14 +23,11 @@ import ( "strings" ) -import ( - perrors "github.com/pkg/errors" -) - import ( "dubbo.apache.org/dubbo-go/v3/cluster/router" "dubbo.apache.org/dubbo-go/v3/common" "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/config_center" "dubbo.apache.org/dubbo-go/v3/protocol" "dubbo.apache.org/dubbo-go/v3/remoting/xds" @@ -44,14 +41,14 @@ const ( // MeshRouter have type MeshRouter struct { - client *xds.WrappedClient + client *xds.WrappedClientImpl } // NewMeshRouter construct an NewConnCheckRouter via url func NewMeshRouter() (router.PriorityRouter, error) { xdsWrappedClient := xds.GetXDSWrappedClient() if xdsWrappedClient == nil { - return nil, perrors.Errorf("[Mesh Router] xds wrapped client is not created.") + logger.Debugf("[Mesh Router] xds wrapped client is not created.") } return &MeshRouter{ client: xdsWrappedClient, @@ -60,6 +57,9 @@ func NewMeshRouter() (router.PriorityRouter, error) { // Route gets a list of routed invoker func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocation protocol.Invocation) []protocol.Invoker { + if r.client == nil { + return invokers + } hostAddr, err := r.client.GetHostAddrByServiceUniqueKey(getSubscribeName(url)) if err != nil { // todo deal with error diff --git a/filter/xds/cb/filter.go b/filter/xds/cb/filter.go index 7ddf947fcd..50631bd809 100644 --- a/filter/xds/cb/filter.go +++ b/filter/xds/cb/filter.go @@ -41,7 +41,7 @@ func init() { // if you wish to using opentracing, please add the this filter into your filter attribute in your configure file. // notice that this could be used in both client-side and server-side. type circuitBreakerFilter struct { - client *xds.WrappedClient + client xds.XDSWrapperClient } func (cb *circuitBreakerFilter) Invoke(ctx context.Context, invoker protocol.Invoker, invocation protocol.Invocation) protocol.Result { diff --git a/go.mod b/go.mod index 34d39834cc..47a910a68b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module dubbo.apache.org/dubbo-go/v3 -go 1.15 +go 1.17 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.0 @@ -56,3 +56,116 @@ require ( k8s.io/apimachinery v0.22.4 k8s.io/client-go v0.16.9 ) + +require ( + cloud.google.com/go v0.65.0 // indirect + github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/aliyun/alibaba-cloud-sdk-go v1.61.18 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/bits-and-blooms/bitset v1.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect + github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dustin/go-humanize v1.0.0 // indirect + github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect + github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect + github.com/go-errors/errors v1.0.1 // indirect + github.com/go-kit/log v0.1.0 // indirect + github.com/go-logfmt/logfmt v0.5.0 // indirect + github.com/go-logr/logr v0.4.0 // indirect + github.com/go-ole/go-ole v1.2.4 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/btree v1.0.1 // indirect + github.com/google/gofuzz v1.1.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/websocket v1.4.2 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jonboulle/clockwork v0.2.2 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/k0kubun/pp v3.0.1+incompatible // indirect + github.com/leodido/go-urn v1.2.1 // indirect + github.com/mattn/go-colorable v0.1.7 // indirect + github.com/mattn/go-isatty v0.0.14 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/pelletier/go-toml v1.7.0 // indirect + github.com/pierrec/lz4 v2.5.2+incompatible // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.32.1 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/statsd_exporter v0.21.0 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/shirou/gopsutil v3.20.11+incompatible // indirect + github.com/shirou/gopsutil/v3 v3.21.6 // indirect + github.com/sirupsen/logrus v1.7.0 // indirect + github.com/soheilhy/cmux v0.1.5 // indirect + github.com/spaolacci/murmur3 v1.1.0 // indirect + github.com/spf13/afero v1.2.2 // indirect + github.com/spf13/cast v1.3.0 // indirect + github.com/spf13/jwalterweatherman v1.0.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/viper v1.7.1 // indirect + github.com/stretchr/objx v0.1.1 // indirect + github.com/subosito/gotenv v1.2.0 // indirect + github.com/tklauser/go-sysconf v0.3.6 // indirect + github.com/tklauser/numcpus v0.2.2 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect + github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect + github.com/uber/jaeger-lib v2.4.1+incompatible // indirect + github.com/ugorji/go/codec v1.2.6 // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect + go.etcd.io/etcd/client/v2 v2.305.2 // indirect + go.etcd.io/etcd/pkg/v3 v3.5.2 // indirect + go.etcd.io/etcd/raft/v3 v3.5.2 // indirect + go.opencensus.io v0.23.0 // indirect + go.opentelemetry.io/contrib v0.20.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect + go.opentelemetry.io/otel v0.20.0 // indirect + go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect + go.opentelemetry.io/otel/metric v0.20.0 // indirect + go.opentelemetry.io/otel/sdk v0.20.0 // indirect + go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect + go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect + go.opentelemetry.io/otel/trace v0.20.0 // indirect + go.opentelemetry.io/proto/otlp v0.7.0 // indirect + go.uber.org/multierr v1.6.0 // indirect + golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect + golang.org/x/net v0.0.0-20211105192438-b53810dc28af // indirect + golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect + golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect + golang.org/x/text v0.3.6 // indirect + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect + google.golang.org/appengine v1.6.6 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/ini.v1 v1.51.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + k8s.io/api v0.16.9 // indirect + k8s.io/klog v1.0.0 // indirect + k8s.io/klog/v2 v2.9.0 // indirect + k8s.io/utils v0.0.0-20190801114015-581e00157fb1 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect + sigs.k8s.io/yaml v1.2.0 // indirect +) diff --git a/go.sum b/go.sum index dc1bbb3fee..de7ab32010 100644 --- a/go.sum +++ b/go.sum @@ -950,7 +950,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -965,7 +964,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -975,7 +973,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1200,7 +1197,6 @@ golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20201014170642-d1624618ad65/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1208,9 +1204,7 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= -gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= -gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= diff --git a/integrate_test.sh b/integrate_test.sh index e2684ffdf0..f8ca769423 100644 --- a/integrate_test.sh +++ b/integrate_test.sh @@ -41,5 +41,7 @@ git clone -b master https://github.com/apache/dubbo-go-samples.git samples && cd # update dubbo-go to current commit id go mod edit -replace=dubbo.apache.org/dubbo-go/v3=github.com/"$1"/v3@"$2" +go mod tidy + # start integrate test ./start_integrate_test.sh diff --git a/registry/mocks/NotifyListener.go b/registry/mocks/NotifyListener.go new file mode 100644 index 0000000000..1e08b5c915 --- /dev/null +++ b/registry/mocks/NotifyListener.go @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by mockery v2.9.4. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" +) + +import ( + registry "dubbo.apache.org/dubbo-go/v3/registry" +) + +// NotifyListener is an autogenerated mock type for the NotifyListener type +type NotifyListener struct { + mock.Mock +} + +// Notify provides a mock function with given fields: _a0 +func (_m *NotifyListener) Notify(_a0 *registry.ServiceEvent) { + _m.Called(_a0) +} + +// NotifyAll provides a mock function with given fields: _a0, _a1 +func (_m *NotifyListener) NotifyAll(_a0 []*registry.ServiceEvent, _a1 func()) { + _m.Called(_a0, _a1) +} diff --git a/registry/xds/registry.go b/registry/xds/registry.go index 78da9d76b4..a0daf8c522 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -35,22 +35,18 @@ import ( "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/registry" "dubbo.apache.org/dubbo-go/v3/remoting/xds" + common2 "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" ) var localIP = "" -const ( - // RegistryConnDelay registry connection delay - RegistryConnDelay = 3 -) - func init() { localIP = common.GetLocalIp() extension.SetRegistry(constant.XDSRegistryKey, newXDSRegistry) } type xdsRegistry struct { - xdsWrappedClient *xds.WrappedClient + xdsWrappedClient xds.XDSWrapperClient registryURL *common.URL } @@ -163,10 +159,11 @@ func newXDSRegistry(url *common.URL) (registry.Registry, error) { pn := os.Getenv(constant.PodNameEnvKey) ns := os.Getenv(constant.PodNamespaceEnvKey) if pn == "" || ns == "" { - return nil, perrors.New("POD_NAME and POD_NAMESPACE can't be empty when using xds registry") + return nil, perrors.Errorf("%s and %s can't be empty when using xds registry", + constant.PodNameEnvKey, constant.PodNamespaceEnvKey) } - wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, xds.NewAddr(url.Ip+":"+url.Port)) + wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, common2.NewAddr(url.Ip+":"+url.Port)) if err != nil { return nil, err } diff --git a/remoting/listener.go b/remoting/listener.go index ea2300fddb..6fd6b07ffd 100644 --- a/remoting/listener.go +++ b/remoting/listener.go @@ -35,7 +35,7 @@ type EventType int const ( // EventTypeAdd means add event - EventTypeAdd = iota + EventTypeAdd EventType = iota // EventTypeDel means del event EventTypeDel // EventTypeUpdate means update event diff --git a/remoting/xds/client.go b/remoting/xds/client.go index c6b5f2fa3d..9325fb7d7c 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -18,52 +18,36 @@ package xds import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - "strings" "sync" "time" ) import ( - v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" - - structpb "github.com/golang/protobuf/ptypes/struct" - perrors "github.com/pkg/errors" - - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" ) import ( - "dubbo.apache.org/dubbo-go/v3/common" "dubbo.apache.org/dubbo-go/v3/common/constant" - "dubbo.apache.org/dubbo-go/v3/common/logger" "dubbo.apache.org/dubbo-go/v3/registry" - "dubbo.apache.org/dubbo-go/v3/remoting" + xdsCommon "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/ewatcher" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/interfaceMapping" "dubbo.apache.org/dubbo-go/v3/xds/client" - "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" ) const ( + // todo make istiodTokenPath configurable + defaultIstiodTokenPath = "/var/run/secrets/token/istio-token" + defaultIstiodDebugPort = "8080" gRPCUserAgentName = "gRPC Go" clientFeatureNoOverprovisioning = "envoy.lb.does_not_support_overprovisioning" ) -const ( - istiodTokenPath = "/var/run/secrets/token/istio-token" - authorizationHeader = "Authorization" - istiodTokenPrefix = "Bearer " -) +// xdsWrappedClient should only init once +var xdsWrappedClient *WrappedClientImpl -var xdsWrappedClient *WrappedClient - -type WrappedClient struct { +type WrappedClientImpl struct { /* local info */ @@ -78,14 +62,14 @@ type WrappedClient struct { /* hostAddr is local pod's cluster and hostAddr, like dubbo-go-app.default.svc.cluster.local:20000 */ - hostAddr string + hostAddr xdsCommon.Addr /* istiod info istiodAddr is istio $(istioSeviceFullName):$(xds-grpc-port) like istiod.istio-system.svc.cluster.local:15010 istiodPodIP is to call istiod unexposed debug port 8080 */ - istiodAddr Addr + istiodAddr xdsCommon.Addr istiodPodIP string /* @@ -94,10 +78,9 @@ type WrappedClient struct { xdsClient client.XDSClient /* - interfaceAppNameMap store map of serviceUniqueKey -> hostAddr + interfaceMapHandler manages dubbogo metadata containing service key -> hostAddr map */ - interfaceAppNameMap map[string]string - interfaceAppNameMapLock sync.RWMutex + interfaceMapHandler interfaceMapping.InterfaceMapHandler /* rdsMap cache router config @@ -134,168 +117,133 @@ type WrappedClient struct { /* hostAddrClusterCtxMap[hostAddr][clusterName] -> endPointWatcherCtx */ - hostAddrClusterCtxMap map[string]map[string]endPointWatcherCtx + hostAddrClusterCtxMap map[string]map[string]ewatcher.EWatcher hostAddrClusterCtxMapLock sync.RWMutex - /* - interfaceNameHostAddrMap cache the dubbo interface unique key -> hostName - the data is read from istiod:8080/debug/adsz, connection metadata["LABELS"]["DUBBO_GO"] - */ - interfaceNameHostAddrMap map[string]string - interfaceNameHostAddrMapLock sync.RWMutex - /* subscribeStopChMap stores subscription stop chan */ subscribeStopChMap sync.Map } -func GetXDSWrappedClient() *WrappedClient { +func GetXDSWrappedClient() *WrappedClientImpl { return xdsWrappedClient } -func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr Addr) (*WrappedClient, error) { +// NewXDSWrappedClient create or get singleton xdsWrappedClient +func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr xdsCommon.Addr) (XDSWrapperClient, error) { // todo @(laurence) safety problem? what if to concurrent 'new' both create new client? if xdsWrappedClient != nil { return xdsWrappedClient, nil } - // get hostname from http://localhost:8080/debug/endpointz - newClient := &WrappedClient{ - podName: podName, - namespace: namespace, - localIP: localIP, - istiodAddr: istioAddr, - interfaceAppNameMap: make(map[string]string), + + // write param + newClient := &WrappedClientImpl{ + podName: podName, + namespace: namespace, + localIP: localIP, + istiodAddr: istioAddr, rdsMap: make(map[string]resource.RouteConfigUpdate), cdsMap: make(map[string]resource.ClusterUpdate), hostAddrListenerMap: make(map[string]map[string]registry.NotifyListener), - hostAddrClusterCtxMap: make(map[string]map[string]endPointWatcherCtx), + hostAddrClusterCtxMap: make(map[string]map[string]ewatcher.EWatcher), - interfaceNameHostAddrMap: make(map[string]string), - cdsUpdateEventChan: make(chan struct{}), - cdsUpdateEventHandlers: make([]func(), 0), + cdsUpdateEventChan: make(chan struct{}), + cdsUpdateEventHandlers: make([]func(), 0), } + + // 1. init xdsclient + if err := newClient.initXDSClient(); err != nil { + return nil, err + } + // 2. watching cds update event // todo @(laurence) gr control - go newClient.runWatchingResource() - if err := newClient.initClientAndLoadLocalHostAddr(); err != nil { + go newClient.runWatchingCdsUpdateEvent() + + // 3. load basic info from istiod and start listening cds + if err := newClient.startWatchingAllClusterAndLoadLocalHostAddrAndIstioPodIP(); err != nil { return nil, err } + // 4. init interface map handler + newClient.interfaceMapHandler = interfaceMapping.NewInterfaceMapHandlerImpl( + newClient.xdsClient, + defaultIstiodTokenPath, + xdsCommon.NewAddr(newClient.istiodPodIP+":"+defaultIstiodDebugPort), + newClient.hostAddr) + xdsWrappedClient = newClient return newClient, nil } -func (w *WrappedClient) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { - req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s:8080/debug/adsz", w.istiodPodIP), nil) - token, err := ioutil.ReadFile(istiodTokenPath) - if err != nil { - return nil, err - } - req.Header.Add(authorizationHeader, istiodTokenPrefix+string(token)) - rsp, err := http.DefaultClient.Do(req) - if err != nil { - logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio %s, IP %s with error %s\n", - w.istiodAddr.HostnameOrIP, w.istiodPodIP, err) - return nil, err - } - - data, err := ioutil.ReadAll(rsp.Body) - if err != nil { - return nil, err - } - adszRsp := &ADSZResponse{} - if err := json.Unmarshal(data, adszRsp); err != nil { - return nil, err - } - return adszRsp.GetMap(), nil +// GetHostAddrByServiceUniqueKey todo 1. timeout 2. hostAddr change? +func (w *WrappedClientImpl) GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) { + return w.interfaceMapHandler.GetHostAddrMap(serviceUniqueKey) } -// GetHostAddrByServiceUniqueKey todo 1. timeout 2. hostAddr change? -func (w *WrappedClient) GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) { - w.interfaceNameHostAddrMapLock.RLock() - if hostAddr, ok := w.interfaceNameHostAddrMap[serviceUniqueKey]; ok { - return hostAddr, nil - } - w.interfaceNameHostAddrMapLock.Unlock() - - for { - if interfaceHostAddrMap, err := w.getServiceUniqueKeyHostAddrMapFromPilot(); err != nil { - return "", err - } else { - w.interfaceNameHostAddrMapLock.Lock() - w.interfaceNameHostAddrMap = interfaceHostAddrMap - w.interfaceNameHostAddrMapLock.Unlock() - hostName, ok := interfaceHostAddrMap[serviceUniqueKey] - if !ok { - logger.Infof("[XDS Wrapped Client] Try getting interface %s 's host from istio %d:8080\n", serviceUniqueKey, w.istiodPodIP) - time.Sleep(time.Millisecond * 100) - continue - } - return hostName, nil - } +// ChangeInterfaceMap change the map of serviceUniqueKey -> appname, if add is true, register, else unregister +func (w *WrappedClientImpl) ChangeInterfaceMap(serviceUniqueKey string, add bool) error { + if add { + return w.interfaceMapHandler.Register(serviceUniqueKey) } + return w.interfaceMapHandler.UnRegister(serviceUniqueKey) } -func getHostNameAndPortFromAddr(hostAddr string) (string, string) { - ipPort := strings.Split(hostAddr, ":") - hostName := ipPort[0] - port := ipPort[1] - return hostName, port +func (w *WrappedClientImpl) GetRouterConfig(hostAddr string) resource.RouteConfigUpdate { + w.rdsMapLock.RLock() + defer w.rdsMapLock.RUnlock() + routeConfig, ok := w.rdsMap[hostAddr] + if ok { + return routeConfig + } + return resource.RouteConfigUpdate{} } -func (w *WrappedClient) GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate { - hostName, port := getHostNameAndPortFromAddr(hostAddr) +func (w *WrappedClientImpl) GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate { + addr := xdsCommon.NewAddr(hostAddr) w.cdsMapLock.RLock() defer w.cdsMapLock.Unlock() for clusterName, v := range w.cdsMap { - clusterNameData := strings.Split(clusterName, "|") - if clusterNameData[1] == port && clusterNameData[3] == hostName { + cluster := xdsCommon.NewCluster(clusterName) + if cluster.Addr.Port == addr.Port && cluster.Addr.HostnameOrIP == addr.HostnameOrIP { return v } } return resource.ClusterUpdate{} } -func (w *WrappedClient) getAllVersionClusterName(hostAddr string) []string { - hostName, port := getHostNameAndPortFromAddr(hostAddr) - allVersionClusterNames := make([]string, 0) - w.cdsMapLock.RLock() - defer w.cdsMapLock.Unlock() - for clusterName := range w.cdsMap { - clusterNameData := strings.Split(clusterName, "|") - if clusterNameData[1] == port && clusterNameData[3] == hostName { - allVersionClusterNames = append(allVersionClusterNames, clusterName) - } +func (w *WrappedClientImpl) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { + _, ok := w.subscribeStopChMap.Load(svcUniqueName) + if ok { + return perrors.Errorf("XDS WrappedClientImpl subscribe interface %s failed, subscription already exist.", interfaceName) } - return allVersionClusterNames + stopCh := make(chan struct{}) + w.subscribeStopChMap.Store(svcUniqueName, stopCh) + w.registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName, lst) + <-stopCh + w.unregisterHostLevelSubscription(hostAddr, svcUniqueName) + return nil } -func generateRegistryEvent(clusterName string, endpoint resource.Endpoint, interfaceName string) *registry.ServiceEvent { - // todo now only support triple protocol - url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", endpoint.Address, interfaceName)) - logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", - interfaceName, endpoint.Address, endpoint.HealthStatus) - clusterNames := strings.Split(clusterName, "|") - // todo const MeshSubsetKey - url.AddParam(constant.MeshSubsetKey, clusterNames[2]) - url.AddParam(constant.MeshClusterIDKey, clusterName) - url.AddParam(constant.MeshHostAddrKey, clusterNames[3]+":"+clusterNames[1]) - if endpoint.HealthStatus == resource.EndpointHealthStatusUnhealthy { - return ®istry.ServiceEvent{ - Action: remoting.EventTypeDel, - Service: url, - } - } - return ®istry.ServiceEvent{ - Action: remoting.EventTypeUpdate, - Service: url, +func (w *WrappedClientImpl) UnSubscribe(svcUniqueName string) { + if stopCh, ok := w.subscribeStopChMap.Load(svcUniqueName); ok { + close(stopCh.(chan struct{})) } + w.subscribeStopChMap.Delete(svcUniqueName) +} + +func (w *WrappedClientImpl) GetHostAddress() xdsCommon.Addr { + return w.hostAddr +} + +func (w *WrappedClientImpl) GetIstioPodIP() string { + return w.istiodPodIP } // registerHostLevelSubscription register: 1. all related cluster, 2. router config -func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName string, lst registry.NotifyListener) { +func (w *WrappedClientImpl) registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName string, lst registry.NotifyListener) { // 1. listen all cluster related endpoint w.hostAddrListenerMapLock.Lock() if _, ok := w.hostAddrListenerMap[hostAddr]; ok { @@ -308,7 +256,7 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s w.hostAddrListenerMap[hostAddr] = make(map[string]registry.NotifyListener) w.hostAddrClusterCtxMapLock.Lock() - w.hostAddrClusterCtxMap[hostAddr] = make(map[string]endPointWatcherCtx) + w.hostAddrClusterCtxMap[hostAddr] = make(map[string]ewatcher.EWatcher) w.hostAddrClusterCtxMapLock.Unlock() w.hostAddrListenerMap[hostAddr][svcUniqueName] = lst @@ -322,7 +270,7 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s // do patch w.hostAddrClusterCtxMapLock.RLock() listeningClustersCancelMap := w.hostAddrClusterCtxMap[hostAddr] - w.hostAddrClusterCtxMapLock.Unlock() + w.hostAddrClusterCtxMapLock.RUnlock() oldlisteningClusterMap := make(map[string]bool) for cluster := range listeningClustersCancelMap { @@ -335,14 +283,10 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s continue } // new cluster - watcher := endPointWatcherCtx{ - interfaceName: interfaceName, - clusterName: updatedClusterName, - hostAddr: hostAddr, - xdsClient: w, - } - cancel := w.xdsClient.WatchEndpoints(updatedClusterName, watcher.handle) - watcher.cancel = cancel + watcher := ewatcher.NewEndpointWatcherCtxImpl( + updatedClusterName, hostAddr, interfaceName, &w.hostAddrListenerMapLock, w.hostAddrListenerMap) + cancel := w.xdsClient.WatchEndpoints(updatedClusterName, watcher.Handle) + watcher.SetCancelFunction(cancel) w.hostAddrClusterCtxMapLock.Lock() w.hostAddrClusterCtxMap[hostAddr][updatedClusterName] = watcher w.hostAddrClusterCtxMapLock.Unlock() @@ -355,7 +299,7 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s w.hostAddrClusterCtxMapLock.Lock() if watchCtx, ok := w.hostAddrClusterCtxMap[hostAddr][cluster]; ok { delete(w.hostAddrClusterCtxMap[hostAddr], cluster) - watchCtx.destroy() + watchCtx.Destroy() } w.hostAddrClusterCtxMapLock.Unlock() } @@ -366,13 +310,9 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s // update cluster of now allVersionedClusterName := w.getAllVersionClusterName(hostAddr) for _, c := range allVersionedClusterName { - watcher := endPointWatcherCtx{ - interfaceName: interfaceName, - clusterName: c, - hostAddr: hostAddr, - xdsClient: w, - } - watcher.cancel = w.xdsClient.WatchEndpoints(c, watcher.handle) + watcher := ewatcher.NewEndpointWatcherCtxImpl( + c, hostAddr, interfaceName, &w.hostAddrListenerMapLock, w.hostAddrListenerMap) + watcher.SetCancelFunction(w.xdsClient.WatchEndpoints(c, watcher.Handle)) w.hostAddrClusterCtxMapLock.Lock() w.hostAddrClusterCtxMap[hostAddr][c] = watcher @@ -391,17 +331,7 @@ func (w *WrappedClient) registerHostLevelSubscription(hostAddr, interfaceName, s }) } -func (w *WrappedClient) GetRouterConfig(hostAddr string) resource.RouteConfigUpdate { - w.rdsMapLock.RLock() - defer w.rdsMapLock.Unlock() - routeConfig, ok := w.rdsMap[hostAddr] - if ok { - return routeConfig - } - return resource.RouteConfigUpdate{} -} - -func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName string) { +func (w *WrappedClientImpl) unregisterHostLevelSubscription(hostAddr, svcUniqueName string) { w.hostAddrListenerMapLock.Lock() defer w.hostAddrListenerMapLock.Unlock() if _, ok := w.hostAddrListenerMap[hostAddr]; ok { @@ -414,7 +344,7 @@ func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName keys := make([]string, 0) w.hostAddrClusterCtxMapLock.Lock() for k, c := range w.hostAddrClusterCtxMap[hostAddr] { - c.destroy() + c.Destroy() keys = append(keys, k) } for _, v := range keys { @@ -425,71 +355,33 @@ func (w *WrappedClient) unregisterHostLevelSubscription(hostAddr, svcUniqueName } } -func (w *WrappedClient) Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error { - _, ok := w.subscribeStopChMap.Load(svcUniqueName) - if ok { - return perrors.Errorf("XDS WrappedClient subscribe interface %s failed, subscription already exist.", interfaceName) - } - stopCh := make(chan struct{}) - w.subscribeStopChMap.Store(svcUniqueName, stopCh) - w.registerHostLevelSubscription(hostAddr, interfaceName, svcUniqueName, lst) - <-stopCh - w.unregisterHostLevelSubscription(hostAddr, svcUniqueName) - return nil -} - -func (w *WrappedClient) UnSubscribe(svcUniqueName string) { - if stopCh, ok := w.subscribeStopChMap.Load(svcUniqueName); ok { - close(stopCh.(chan struct{})) - } - w.subscribeStopChMap.Delete(svcUniqueName) -} - -func (w *WrappedClient) interfaceAppNameMap2String() string { - w.interfaceAppNameMapLock.RLock() - defer w.interfaceAppNameMapLock.Unlock() - data, _ := json.Marshal(w.interfaceAppNameMap) - return string(data) -} - -// ChangeInterfaceMap change the map of serviceUniqueKey -> appname, if add is true, register, else unregister -func (w *WrappedClient) ChangeInterfaceMap(serviceUniqueKey string, add bool) error { - w.interfaceAppNameMapLock.Lock() - defer w.interfaceAppNameMapLock.Unlock() - if add { - w.interfaceAppNameMap[serviceUniqueKey] = w.hostAddr - } else { - delete(w.interfaceAppNameMap, serviceUniqueKey) - } - if w.xdsClient == nil { - xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodAddr) - if err != nil { - return err - } - w.xdsClient = xdsClient - return nil - } - - if err := w.xdsClient.SetMetadata(getDubboGoMetadata(w.interfaceAppNameMap2String())); err != nil { +func (w *WrappedClientImpl) initXDSClient() error { + xdsClient, err := xdsClientFactoryFunction(w.localIP, w.podName, w.namespace, w.istiodAddr) + if err != nil { return err } + w.xdsClient = xdsClient return nil } -func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { +// startWatchingAllClusterAndLoadLocalHostAddrAndIstioPodIP is blocking function +// 1. start watching all cluster by cds +// 2. discovery local pod's hostAddr by cds and eds +// 3. discovery istiod pod ip by cds and eds +func (w *WrappedClientImpl) startWatchingAllClusterAndLoadLocalHostAddrAndIstioPodIP() error { // call watch and refresh istiod debug interface - xdsClient, err := newxdsClient(w.localIP, w.podName, w.namespace, w.interfaceAppNameMap2String(), w.istiodAddr) - if err != nil { - return err - } foundLocalStopCh := make(chan struct{}) foundIstiodStopCh := make(chan struct{}) + discoveryFinishedStopCh := make(chan struct{}) + // todo timeout configure + timeoutCh := time.After(time.Second * 3) foundLocal := false foundIstiod := false var cancel1 func() var cancel2 func() + // todo @(laurence) here, if istiod is unhealthy, here should be timeout and tell developer. - _ = xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { + _ = w.xdsClient.WatchCluster("*", func(update resource.ClusterUpdate, err error) { if update.ClusterName == "" { return } @@ -510,19 +402,17 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { return } // only into here during start sniffing istiod/service prcedure - clusterNameList := strings.Split(update.ClusterName, "|") - // todo: what's going on? istiod can't discover istiod.istio-system.svc.cluster.local!! - if clusterNameList[3] == w.istiodAddr.HostnameOrIP { + cluster := xdsCommon.NewCluster(update.ClusterName) + if cluster.Addr.HostnameOrIP == w.istiodAddr.HostnameOrIP { // 1. find istiod podIP // todo: When would eds level watch be canceled? - cancel1 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + cancel1 = w.xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundIstiod { return } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - addrs := strings.Split(e.Address, ":") - w.istiodPodIP = addrs[0] + w.istiodPodIP = xdsCommon.NewAddr(e.Address).HostnameOrIP foundIstiod = true close(foundLocalStopCh) } @@ -532,16 +422,15 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { } // 2. found local hostAddr // todo: When would eds level watch be canceled? - cancel2 = xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { + cancel2 = w.xdsClient.WatchEndpoints(update.ClusterName, func(endpoint resource.EndpointsUpdate, err error) { if foundLocal { return } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - addrs := strings.Split(e.Address, ":") - if addrs[0] == w.localIP { - clusterNames := strings.Split(update.ClusterName, "|") - w.hostAddr = clusterNames[3] + ":" + clusterNames[1] + if xdsCommon.NewAddr(e.Address).HostnameOrIP == w.localIP { + cluster := xdsCommon.NewCluster(update.ClusterName) + w.hostAddr = cluster.Addr foundLocal = true close(foundIstiodStopCh) } @@ -549,71 +438,73 @@ func (w *WrappedClient) initClientAndLoadLocalHostAddr() error { } }) }) - <-foundIstiodStopCh - <-foundLocalStopCh - cancel1() - cancel2() - w.xdsClient = xdsClient - return nil -} - -func newxdsClient(localIP, podName, namespace, dubboGoMetadata string, istioAddr Addr) (client.XDSClient, error) { - // todo fix these ugly magic num - v3NodeProto := &v3corepb.Node{ - Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", - UserAgentName: gRPCUserAgentName, - UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, - ClientFeatures: []string{clientFeatureNoOverprovisioning}, - } - - nonNilCredsConfigV2 := &bootstrap.Config{ - XDSServer: &bootstrap.ServerConfig{ - ServerURI: istioAddr.String(), - Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), - TransportAPI: version.TransportV3, - NodeProto: v3NodeProto, - }, - ClientDefaultListenerResourceNameTemplate: "%s", - } - - newClient, err := client.NewWithConfig(nonNilCredsConfigV2) - if err != nil { - return nil, err - } - if err := newClient.SetMetadata(getDubboGoMetadata(dubboGoMetadata)); err != nil { - return nil, err - } - return newClient, nil -} - -func getDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { - return &structpb.Struct{ - Fields: map[string]*structpb.Value{ - constant.XDSMetadataClusterIDKey: { - // Set cluster id to Kubernetes to ensure dubbo-go's xds client can get service - // istiod.istio-system.svc.cluster.local's - // pods ip from istiod by eds, to call no-endpoint port of istio like 8080 - Kind: &structpb.Value_StringValue{StringValue: constant.XDSMetadataDefaultDomainName}, - }, - constant.XDSMetadataLabelsKey: { - Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ - Fields: map[string]*structpb.Value{ - constant.XDSMetadataDubboGoMapperKey: { - Kind: &structpb.Value_StringValue{StringValue: dubboGoMetadata}, - }, - }, - }}, - }, - }, + go func() { + <-foundIstiodStopCh + <-foundLocalStopCh + close(discoveryFinishedStopCh) + }() + + select { + case <-discoveryFinishedStopCh: + // discovery success + // waiting for cancel function to have value + time.Sleep(time.Second) + cancel1() + cancel2() + return nil + case <-timeoutCh: + if cancel1 != nil { + cancel1() + } + if cancel2 != nil { + cancel2() + } + select { + case <-foundIstiodStopCh: + return DiscoverLocalError + default: + return DiscoverIstioPodError + } } } -func (w *WrappedClient) runWatchingResource() { +// runWatchingCdsUpdateEvent is blocking function, starts to read event from cdsUpdateEventChan and call cdsUpdateEventHandlers +func (w *WrappedClientImpl) runWatchingCdsUpdateEvent() { for range w.cdsUpdateEventChan { w.cdsUpdateEventHandlersLock.RLock() for _, h := range w.cdsUpdateEventHandlers { h() } - w.cdsUpdateEventHandlersLock.Unlock() + w.cdsUpdateEventHandlersLock.RUnlock() } } + +// getAllVersionClusterName get all clusterID that is the subset of given hostAddr from cache: cdsMap +// like: if given hostAddr is 'outbound|20000||dubbo-go-app.default.svc.cluster.local', and result would be +// ['outbound|20000|v1|dubbo-go-app.default.svc.cluster.local', +// 'outbound|20000||dubbo-go-app.default.svc.cluster.local', +// 'outbound|20000|v2|dubbo-go-app.default.svc.cluster.local'] +func (w *WrappedClientImpl) getAllVersionClusterName(hostAddr string) []string { + addr := xdsCommon.NewAddr(hostAddr) + allVersionClusterNames := make([]string, 0) + w.cdsMapLock.RLock() + defer w.cdsMapLock.RUnlock() + for clusterName, _ := range w.cdsMap { + cluster := xdsCommon.NewCluster(clusterName) + if cluster.Addr.Port == addr.Port && cluster.Addr.HostnameOrIP == addr.HostnameOrIP { + allVersionClusterNames = append(allVersionClusterNames, clusterName) + } + } + return allVersionClusterNames +} + +type XDSWrapperClient interface { + Subscribe(svcUniqueName, interfaceName, hostAddr string, lst registry.NotifyListener) error + UnSubscribe(svcUniqueName string) + GetRouterConfig(hostAddr string) resource.RouteConfigUpdate + GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) + ChangeInterfaceMap(serviceUniqueKey string, add bool) error + GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate + GetHostAddress() xdsCommon.Addr + GetIstioPodIP() string +} diff --git a/remoting/xds/client_test.go b/remoting/xds/client_test.go new file mode 100644 index 0000000000..60fc5e86d1 --- /dev/null +++ b/remoting/xds/client_test.go @@ -0,0 +1,764 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + "testing" + "time" +) + +import ( + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + + "go.uber.org/atomic" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/registry" + registryMocks "dubbo.apache.org/dubbo-go/v3/registry/mocks" + "dubbo.apache.org/dubbo-go/v3/remoting" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/mocks" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +const ( + dubbogoPortFoo = "20000" + istioXDSPortFoo = "15010" + + podNameFoo = "mockPodName" + localNamespaceFoo = "default" + + localIPFoo = "172.16.100.1" + istioIPFoo = "172.16.100.2" + + localAddrFoo = localIPFoo + ":" + dubbogoPortFoo + istioXDSAddrFoo = istioIPFoo + ":" + istioXDSPortFoo + + istioHostNameFoo = "istiod.istio-system.svc.cluster.local" + istioHostAddrFoo = istioHostNameFoo + ":" + istioXDSPortFoo + localHostNameFoo = "dubbo-go-app." + localNamespaceFoo + ".svc.cluster.local" + localHostAddrFoo = localHostNameFoo + ":" + dubbogoPortFoo + + istioClusterNameFoo = "outbound|" + istioXDSPortFoo + "||" + istioHostNameFoo + localClusterNameFoo = "outbound|" + dubbogoPortFoo + "||" + localHostNameFoo +) + +const ( + providerIPFoo = "172.16.100.3" + providerV1IPFoo = "172.16.100.4" + providerV2IPFoo = "172.16.100.5" + providerAddrFoo = providerIPFoo + ":" + dubbogoPortFoo + providerV1AddrFoo = providerV1IPFoo + ":" + dubbogoPortFoo + providerV2AddrFoo = providerV2IPFoo + ":" + dubbogoPortFoo + + dubbogoProviderHostName = "dubbo-go-app-provider." + localNamespaceFoo + ".svc.cluster.local" + dubbogoProviderHostAddrFoo = dubbogoProviderHostName + ":" + dubbogoPortFoo + + dubbogoProviderInterfaceNameFoo = "api.Greeter" + dubbogoProviderserivceUniqueKeyFoo = "provider::" + dubbogoProviderInterfaceNameFoo + + providerClusterNameFoo = "outbound|" + dubbogoPortFoo + "||" + dubbogoProviderHostName + providerClusterV1NameFoo = "outbound|" + dubbogoPortFoo + "|v1|" + dubbogoProviderHostName + providerClusterV2NameFoo = "outbound|" + dubbogoPortFoo + "|v2|" + dubbogoProviderHostName +) + +func TestWrappedClientImpl(t *testing.T) { + // test New WrappedClientImpl + testFailedWithIstioCDS(t) + testFailedWithLocalCDS(t) + testFailedWithNoneCDS(t) + + testFailedWithLocalEDSFailed(t) + testFailedWithIstioEDSFailed(t) + + testWithDiscoverySuccess(t) + + assert.NotNil(t, GetXDSWrappedClient()) + + // test Subscription + testSubscribe(t) +} + +func testWithDiscoverySuccess(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + // istioClusterName CDS, this and next calling must be async because testify.Mock.MethodCalled() has calling lock + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: istioClusterNameFoo, + }, nil) + + // localClusterName CDS + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: localClusterNameFoo, + }, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Nil(t, err) + assert.NotNil(t, xdsWrappedClient) + + // assert eds cancel is called + assert.Equal(t, int32(2), cancelCalledCounter.Load()) + // discovery p + assert.Equal(t, istioIPFoo, xdsWrappedClient.GetIstioPodIP()) + address := xdsWrappedClient.GetHostAddress() + assert.Equal(t, localHostAddrFoo, address.String()) +} + +func testFailedWithIstioCDS(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + // do not send istioClusterName message, which cause discover istio ip failed + //go clusterUpdateHandler(resource.ClusterUpdate{ + // ClusterName: istioClusterNameFoo, + //}, nil) + + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: localClusterNameFoo, + }, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Equal(t, DiscoverIstioPodError, err) + assert.Nil(t, xdsWrappedClient) + assert.Equal(t, int32(1), cancelCalledCounter.Load()) +} + +func testFailedWithLocalCDS(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: istioClusterNameFoo, + }, nil) + + // do not send localClusterNameFoo cds message, which cause discover local addr failed + //go clusterUpdateHandler(resource.ClusterUpdate{ + // ClusterName: localClusterNameFoo, + //}, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Equal(t, DiscoverLocalError, err) + assert.Nil(t, xdsWrappedClient) + assert.Equal(t, int32(1), cancelCalledCounter.Load()) +} + +func testFailedWithNoneCDS(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + // do not send any cds message, which cause discover failed + //go clusterUpdateHandler(resource.ClusterUpdate{ + // ClusterName: istioClusterNameFoo, + //}, nil) + + //go clusterUpdateHandler(resource.ClusterUpdate{ + // ClusterName: localClusterNameFoo, + //}, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Equal(t, DiscoverIstioPodError, err) + assert.Nil(t, xdsWrappedClient) + assert.Equal(t, int32(0), cancelCalledCounter.Load()) +} + +func testFailedWithLocalEDSFailed(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: istioClusterNameFoo, + }, nil) + + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: localClusterNameFoo, + }, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + // do not send local eds message + //clusterUpdateHandler(resource.EndpointsUpdate{ + // Localities: []resource.Locality{ + // { + // Endpoints: []resource.Endpoint{ + // { + // Address: localAddrFoo, + // }, + // }, + // }, + // }, + //}, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Equal(t, DiscoverLocalError, err) + assert.Nil(t, xdsWrappedClient) + assert.Equal(t, int32(2), cancelCalledCounter.Load()) +} + +func testFailedWithIstioEDSFailed(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + cancelCalledCounter := &atomic.Int32{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: istioClusterNameFoo, + }, nil) + + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: localClusterNameFoo, + }, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == istioClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + // do not send istio eds message + //clusterUpdateHandler(resource.EndpointsUpdate{ + // Localities: []resource.Locality{ + // { + // Endpoints: []resource.Endpoint{ + // { + // Address: istioXDSAddrFoo, + // }, + // }, + // }, + // }, + //}, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + return clusterName == localClusterNameFoo + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() { + // return cancel function + cancelCalledCounter.Inc() + }) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Equal(t, DiscoverIstioPodError, err) + assert.Nil(t, xdsWrappedClient) + assert.Equal(t, int32(2), cancelCalledCounter.Load()) +} + +func testSubscribe(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + // all cluster CDS + mockXDSClient.On("WatchCluster", "*", + mock.MatchedBy(func(clusterUpdateHandler func(update resource.ClusterUpdate, err error)) bool { + // istio ClusterName CDS, this and next calling must be async because testify.Mock.MethodCalled() has calling lock + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: istioClusterNameFoo, + }, nil) + + // local ClusterName CDS + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: localClusterNameFoo, + }, nil) + + // provider Cluster Name CDS + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: providerClusterNameFoo, + }, nil) + + // provider Cluster Name v1 CDS + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: providerClusterV1NameFoo, + }, nil) + + // provider Cluster Name v2 CDS + go clusterUpdateHandler(resource.ClusterUpdate{ + ClusterName: providerClusterV2NameFoo, + }, nil) + return true + })). + Return(func() {}) + + // istio cluster EDS + clusterMatch := false + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + clusterMatch = clusterName == istioClusterNameFoo + return clusterMatch + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + if !clusterMatch { + return false + } + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: istioXDSAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() {}) + + // local cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + clusterMatch = clusterName == localClusterNameFoo + return clusterMatch + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + if !clusterMatch { + return false + } + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: localAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() {}) + + // provider cluster EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + clusterMatch = clusterName == providerClusterNameFoo + return clusterMatch + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + if !clusterMatch { + return false + } + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: providerAddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() {}) + + // provider cluster v1 EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + clusterMatch = clusterName == providerClusterV1NameFoo + return clusterMatch + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + if !clusterMatch { + return false + } + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: providerV1AddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() {}) + + // todo test router RDS + mockXDSClient.On("WatchRouteConfig", mock.Anything, mock.Anything).Return(func() {}) + + // provider cluster v2 EDS + mockXDSClient.On("WatchEndpoints", + mock.MatchedBy(func(clusterName string) bool { + clusterMatch = clusterName == providerClusterV2NameFoo + return clusterMatch + }), + mock.MatchedBy(func(clusterUpdateHandler func(update resource.EndpointsUpdate, err error)) bool { + if !clusterMatch { + return false + } + clusterUpdateHandler(resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: providerV2AddrFoo, + }, + }, + }, + }, + }, nil) + return true + })). + Return(func() {}) + + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + return mockXDSClient, nil + } + + xdsWrappedClient = nil + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + assert.Nil(t, err) + assert.NotNil(t, xdsWrappedClient) + + notifyListener := ®istryMocks.NotifyListener{} + notifyListener.On("Notify", mock.Anything).Return(nil) + + go xdsWrappedClient.Subscribe(dubbogoProviderserivceUniqueKeyFoo, dubbogoProviderInterfaceNameFoo, dubbogoProviderHostAddrFoo, notifyListener) + + time.Sleep(time.Second) + notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { + if event.Action == remoting.EventTypeUpdate && + event.Service.Ip == providerV2IPFoo && + event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterV2NameFoo { + return true + } + return false + })) + + notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { + if event.Action == remoting.EventTypeUpdate && + event.Service.Ip == providerV1IPFoo && + event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterV1NameFoo { + return true + } + return false + })) + + notifyListener.AssertCalled(t, "Notify", mock.MatchedBy(func(event *registry.ServiceEvent) bool { + if event.Action == remoting.EventTypeUpdate && + event.Service.Ip == providerIPFoo && + event.Service.GetParam(constant.MeshClusterIDKey, "") == providerClusterNameFoo { + return true + } + return false + })) +} + +// todo TestDestroy +// todo TestRDS diff --git a/remoting/xds/model.go b/remoting/xds/common/model.go similarity index 98% rename from remoting/xds/model.go rename to remoting/xds/common/model.go index a053007d37..e3532ee19e 100644 --- a/remoting/xds/model.go +++ b/remoting/xds/common/model.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package xds +package common import ( "strings" diff --git a/remoting/xds/error.go b/remoting/xds/error.go new file mode 100644 index 0000000000..ca81febdfe --- /dev/null +++ b/remoting/xds/error.go @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + "errors" +) + +var ( + DiscoverLocalError = errors.New("Discovery local Pod's host from xds failed, please register service with endpoint to k8s") + DiscoverIstioPodError = errors.New("Discovery local Pod's host from xds failed, please register service with endpoint to k8s") +) diff --git a/remoting/xds/ewatcher.go b/remoting/xds/ewatcher.go deleted file mode 100644 index 67b3d83438..0000000000 --- a/remoting/xds/ewatcher.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package xds - -import ( - "dubbo.apache.org/dubbo-go/v3/common/constant" - "dubbo.apache.org/dubbo-go/v3/xds/client/resource" -) - -// endPointWatcherCtx is endpoint watching context -type endPointWatcherCtx struct { - clusterName string - interfaceName string - hostAddr string - xdsClient *WrappedClient - cancel func() -} - -// handle handles endpoint update event and send to directory to refresh invoker -func (watcher *endPointWatcherCtx) handle(update resource.EndpointsUpdate, err error) { - for _, v := range update.Localities { - for _, e := range v.Endpoints { - event := generateRegistryEvent(watcher.clusterName, e, watcher.interfaceName) - watcher.xdsClient.hostAddrListenerMapLock.RLock() - for _, l := range watcher.xdsClient.hostAddrListenerMap[watcher.hostAddr] { - // notify all listeners listening this hostAddr - l.Notify(event) - } - watcher.xdsClient.hostAddrListenerMapLock.Unlock() - } - } -} - -// destroy call cancel and send event to listener to remove related invokers of current deleated cluster -func (watcher *endPointWatcherCtx) destroy() { - watcher.cancel() - /* - directory would identify this by EndpointHealthStatusUnhealthy and Location == "*" and none empty clusterId - and delete related invokers - */ - event := generateRegistryEvent(watcher.clusterName, resource.Endpoint{ - HealthStatus: resource.EndpointHealthStatusUnhealthy, - Address: constant.MeshAnyAddrMatcher, // destroy all endpoint of this cluster - }, watcher.interfaceName) - watcher.xdsClient.hostAddrListenerMapLock.RLock() - for _, l := range watcher.xdsClient.hostAddrListenerMap[watcher.hostAddr] { - // notify all listeners listening this hostAddr - l.Notify(event) - } - watcher.xdsClient.hostAddrListenerMapLock.Unlock() -} diff --git a/remoting/xds/ewatcher/ewatcher.go b/remoting/xds/ewatcher/ewatcher.go new file mode 100644 index 0000000000..fabda71831 --- /dev/null +++ b/remoting/xds/ewatcher/ewatcher.go @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ewatcher + +import ( + "fmt" + "sync" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common" + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/registry" + "dubbo.apache.org/dubbo-go/v3/remoting" + xdsCommon "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// endPointWatcherCtx is endpoint watching context +type endPointWatcherCtx struct { + clusterName string + interfaceName string + hostAddr string + + cancel func() + /* + hostAddrListenerMap[hostAddr][serviceUniqueKey] -> registry.NotifyListener + stores all directory listener, which receives events and refresh invokers + */ + hostAddrListenerMap map[string]map[string]registry.NotifyListener + hostAddrListenerMapLock *sync.RWMutex +} + +func NewEndpointWatcherCtxImpl( + clusterName, + hostAddr, + interfaceName string, + hostAddrListenerMapLock *sync.RWMutex, + hostAddrListenerMap map[string]map[string]registry.NotifyListener) EWatcher { + return &endPointWatcherCtx{ + clusterName: clusterName, + hostAddr: hostAddr, + interfaceName: interfaceName, + hostAddrListenerMapLock: hostAddrListenerMapLock, + hostAddrListenerMap: hostAddrListenerMap, + } +} + +func (watcher *endPointWatcherCtx) SetCancelFunction(cancel func()) { + watcher.cancel = cancel +} + +// Handle handles endpoint update event and send to directory to refresh invoker +func (watcher *endPointWatcherCtx) Handle(update resource.EndpointsUpdate, err error) { + for _, v := range update.Localities { + for _, e := range v.Endpoints { + event := generateRegistryEvent(watcher.clusterName, e, watcher.interfaceName) + watcher.hostAddrListenerMapLock.RLock() + for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + // notify all listeners listening this hostAddr + l.Notify(event) + } + watcher.hostAddrListenerMapLock.RUnlock() + } + } +} + +// Destroy call cancel and send event to listener to remove related invokers of current deleated cluster +func (watcher *endPointWatcherCtx) Destroy() { + if watcher.cancel != nil { + watcher.cancel() + } + /* + directory would identify this by EndpointHealthStatusUnhealthy and Location == "*" and none empty clusterId + and delete related invokers + */ + event := generateRegistryEvent(watcher.clusterName, resource.Endpoint{ + HealthStatus: resource.EndpointHealthStatusUnhealthy, + Address: constant.MeshAnyAddrMatcher, // Destroy all endpoint of this cluster + }, watcher.interfaceName) + watcher.hostAddrListenerMapLock.RLock() + for _, l := range watcher.hostAddrListenerMap[watcher.hostAddr] { + // notify all listeners listening this hostAddr + l.Notify(event) + } + watcher.hostAddrListenerMapLock.RUnlock() +} + +func generateRegistryEvent(clusterID string, endpoint resource.Endpoint, interfaceName string) *registry.ServiceEvent { + // todo now only support triple protocol + url, _ := common.NewURL(fmt.Sprintf("tri://%s/%s", endpoint.Address, interfaceName)) + logger.Infof("[XDS Registry] Get Update event from pilot: interfaceName = %s, addr = %s, healthy = %d\n", + interfaceName, endpoint.Address, endpoint.HealthStatus) + cluster := xdsCommon.NewCluster(clusterID) + // todo const MeshSubsetKey + url.AddParam(constant.MeshSubsetKey, cluster.Subset) + url.AddParam(constant.MeshClusterIDKey, clusterID) + url.AddParam(constant.MeshHostAddrKey, cluster.Addr.String()) + if endpoint.HealthStatus == resource.EndpointHealthStatusUnhealthy { + return ®istry.ServiceEvent{ + Action: remoting.EventTypeDel, + Service: url, + } + } + return ®istry.ServiceEvent{ + Action: remoting.EventTypeUpdate, + Service: url, + } +} + +type EWatcher interface { + Destroy() + Handle(update resource.EndpointsUpdate, err error) + SetCancelFunction(cancel func()) +} diff --git a/remoting/xds/ewatcher/ewatcher_test.go b/remoting/xds/ewatcher/ewatcher_test.go new file mode 100644 index 0000000000..651810e713 --- /dev/null +++ b/remoting/xds/ewatcher/ewatcher_test.go @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ewatcher + +import ( + "sync" + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/registry" + mockRegistry "dubbo.apache.org/dubbo-go/v3/registry/mocks" + "dubbo.apache.org/dubbo-go/v3/remoting" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +const ( + clusterNameFoo = "outbound|20000||dubbo-go-app.svc.cluster.local" + hostAddrFoo = "dubbo-go-app.svc.cluster.local:20000" + hostNameFoo = "dubbo-go-app.svc.cluster.local" + hostPortFoo = "20000" + interfaceNameFoo = "api.Greeter" + serviceKeyFoo = "provider::api.Greeter" +) + +var ( + endpointHealthyUpdate = &resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: hostAddrFoo, + HealthStatus: resource.EndpointHealthStatusHealthy, + }, + }, + }, + }, + } + + endpointUnHealthyUpdate = &resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: hostAddrFoo, + HealthStatus: resource.EndpointHealthStatusUnhealthy, + }, + }, + }, + }, + } + + endpointUnknownUpdate = &resource.EndpointsUpdate{ + Localities: []resource.Locality{ + { + Endpoints: []resource.Endpoint{ + { + Address: hostAddrFoo, + HealthStatus: resource.EndpointHealthStatusUnknown, + }, + }, + }, + }, + } +) + +func TestNewEWatcherAndSetCancelFunction(t *testing.T) { + hostAddrListenerMapFoo := map[string]map[string]registry.NotifyListener{ + hostAddrFoo: { + serviceKeyFoo: &mockRegistry.NotifyListener{}, + }, + } + ewatcher := NewEndpointWatcherCtxImpl(clusterNameFoo, hostAddrFoo, interfaceNameFoo, &sync.RWMutex{}, hostAddrListenerMapFoo) + assert.NotNil(t, ewatcher) + + cancel := func() {} + ewatcher.SetCancelFunction(cancel) +} + +func TestNewEWatcherHandle(t *testing.T) { + mockListener := newMockListener() + hostAddrListenerMapFoo := map[string]map[string]registry.NotifyListener{ + hostAddrFoo: { + serviceKeyFoo: mockListener, + }, + } + ewatcher := NewEndpointWatcherCtxImpl(clusterNameFoo, hostAddrFoo, interfaceNameFoo, &sync.RWMutex{}, hostAddrListenerMapFoo) + assert.NotNil(t, ewatcher) + + ewatcher.Handle(*endpointHealthyUpdate, nil) + assert.Equal(t, 1, len(mockListener.Events)) + assert.Equal(t, remoting.EventTypeUpdate, mockListener.Events[0].Action) + assert.Equal(t, hostNameFoo, mockListener.Events[0].Service.Ip) + assert.Equal(t, hostPortFoo, mockListener.Events[0].Service.Port) + + ewatcher.Handle(*endpointUnknownUpdate, nil) + assert.Equal(t, 2, len(mockListener.Events)) + assert.Equal(t, remoting.EventTypeUpdate, mockListener.Events[1].Action) + assert.Equal(t, hostNameFoo, mockListener.Events[1].Service.Ip) + assert.Equal(t, hostPortFoo, mockListener.Events[1].Service.Port) + + ewatcher.Handle(*endpointUnHealthyUpdate, nil) + assert.Equal(t, 3, len(mockListener.Events)) + assert.Equal(t, remoting.EventTypeDel, mockListener.Events[2].Action) + assert.Equal(t, hostNameFoo, mockListener.Events[2].Service.Ip) + assert.Equal(t, hostPortFoo, mockListener.Events[2].Service.Port) + +} + +func TestNewEWatcherDestroy(t *testing.T) { + mockListener := newMockListener() + hostAddrListenerMapFoo := map[string]map[string]registry.NotifyListener{ + hostAddrFoo: { + serviceKeyFoo: mockListener, + }, + } + ewatcher := NewEndpointWatcherCtxImpl(clusterNameFoo, hostAddrFoo, interfaceNameFoo, &sync.RWMutex{}, hostAddrListenerMapFoo) + assert.NotNil(t, ewatcher) + + ewatcher.SetCancelFunction(func() {}) + ewatcher.Destroy() + + assert.Equal(t, len(mockListener.Events), 1) + assert.Equal(t, mockListener.Events[0].Action, remoting.EventTypeDel) + assert.Equal(t, mockListener.Events[0].Service.Location, constant.MeshAnyAddrMatcher) + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshSubsetKey, ""), "") + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshClusterIDKey, ""), clusterNameFoo) + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshHostAddrKey, ""), hostAddrFoo) +} + +func TestNewEWatcherDestroyWithNilCancel(t *testing.T) { + mockListener := newMockListener() + hostAddrListenerMapFoo := map[string]map[string]registry.NotifyListener{ + hostAddrFoo: { + serviceKeyFoo: mockListener, + }, + } + ewatcher := NewEndpointWatcherCtxImpl(clusterNameFoo, hostAddrFoo, interfaceNameFoo, &sync.RWMutex{}, hostAddrListenerMapFoo) + assert.NotNil(t, ewatcher) + + // test nil cancel + ewatcher.Destroy() + + assert.Equal(t, len(mockListener.Events), 1) + assert.Equal(t, mockListener.Events[0].Action, remoting.EventTypeDel) + assert.Equal(t, mockListener.Events[0].Service.Location, constant.MeshAnyAddrMatcher) + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshSubsetKey, ""), "") + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshClusterIDKey, ""), clusterNameFoo) + assert.Equal(t, mockListener.Events[0].Service.GetParam(constant.MeshHostAddrKey, ""), hostAddrFoo) +} + +func TestGenerateRegistryEvent(t *testing.T) { + // todo Add test + +} + +func newMockListener() *mockListener { + return &mockListener{ + Events: make([]*registry.ServiceEvent, 0), + } +} + +type mockListener struct { + Events []*registry.ServiceEvent +} + +func (m *mockListener) Notify(event *registry.ServiceEvent) { + m.Events = append(m.Events, event) +} + +func (m *mockListener) NotifyAll(events []*registry.ServiceEvent, f func()) { + m.Events = append(m.Events, events...) +} diff --git a/remoting/xds/ewatcher/mocks/EWatcher.go b/remoting/xds/ewatcher/mocks/EWatcher.go new file mode 100644 index 0000000000..f531871f57 --- /dev/null +++ b/remoting/xds/ewatcher/mocks/EWatcher.go @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by mockery v2.9.4. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" +) + +import ( + resource "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// EWatcher is an autogenerated mock type for the EWatcher type +type EWatcher struct { + mock.Mock +} + +// Destroy provides a mock function with given fields: +func (_m *EWatcher) Destroy() { + _m.Called() +} + +// Handle provides a mock function with given fields: update, err +func (_m *EWatcher) Handle(update resource.EndpointsUpdate, err error) { + _m.Called(update, err) +} + +// SetCancelFunction provides a mock function with given fields: cancel +func (_m *EWatcher) SetCancelFunction(cancel func()) { + _m.Called(cancel) +} diff --git a/remoting/xds/debug.go b/remoting/xds/interfaceMapping/debug.go similarity index 96% rename from remoting/xds/debug.go rename to remoting/xds/interfaceMapping/debug.go index ed8560268c..b121df5c1f 100644 --- a/remoting/xds/debug.go +++ b/remoting/xds/interfaceMapping/debug.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package xds +package interfaceMapping import ( "encoding/json" @@ -34,6 +34,7 @@ func (a *ADSZResponse) GetMap() map[string]string { result := make(map[string]string) for _, c := range a.Clients { resultMap := make(map[string]string) + // todo assert failed panic _ = json.Unmarshal([]byte(c.Metadata["LABELS"].(map[string]interface{})["DUBBO_GO"].(string)), &resultMap) for k, v := range resultMap { result[k] = v diff --git a/remoting/xds/interfaceMapping/debug_test.go b/remoting/xds/interfaceMapping/debug_test.go new file mode 100644 index 0000000000..e1e20ccdd2 --- /dev/null +++ b/remoting/xds/interfaceMapping/debug_test.go @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package interfaceMapping + +import ( + "encoding/json" + "fmt" + "testing" +) + +import ( + "github.com/stretchr/testify/assert" +) + +const ( + debugAdszDataFoo = `{"totalClients":2,"clients":[{"connectionId":"dubbo-go-app-0.0.1-77b8cd56f9-4hpmb.default-4","connectedAt":"2022-03-23T13:32:49.884373692Z","address":"172.17.80.28:57150","metadata":{"LABELS":{"DUBBO_GO":"{\"providers:api.Greeter::\":\"dubbo-go-app.default.svc.cluster.local:20000\",\"providers:grpc.reflection.v1alpha.ServerReflection::\":\"dubbo-go-app.default.svc.cluster.local:20000\"}","topology.istio.io/cluster":"Kubernetes","topology.kubernetes.io/region":"cn-hangzhou","topology.kubernetes.io/zone":"cn-hangzhou-b"},"CLUSTER_ID":"Kubernetes"},"watches":{"type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment":["outbound|443||istiod.istio-system.svc.cluster.local","outbound|443||metrics-server.kube-system.svc.cluster.local","outbound|80||heapster.kube-system.svc.cluster.local","outbound|443||storage-crd-validate-service.kube-system.svc.cluster.local","outbound|20000||laurence-svc.default.svc.cluster.local","outbound|11280||storage-monitor-service.kube-system.svc.cluster.local","outbound|15010||istiod.istio-system.svc.cluster.local","outbound|443||kubernetes.default.svc.cluster.local","outbound|9153||kube-dns.kube-system.svc.cluster.local","outbound|53||kube-dns.kube-system.svc.cluster.local","","outbound|15012||istiod.istio-system.svc.cluster.local","outbound|8848||nacos.default.svc.cluster.local","outbound|15014||istiod.istio-system.svc.cluster.local","outbound|20000||dubbo-go-app.default.svc.cluster.local"]}},{"connectionId":"dubbo-go-app-0.0.2-f465d67f7-k5ckq.default-2","connectedAt":"2022-03-23T13:32:15.790703481Z","address":"172.17.80.29:35406","metadata":{"LABELS":{"DUBBO_GO":"{\"providers:api.Greeter::\":\"dubbo-go-app.default.svc.cluster.local:20000\",\"providers:grpc.reflection.v1alpha.ServerReflection::\":\"dubbo-go-app.default.svc.cluster.local:20000\"}","topology.istio.io/cluster":"Kubernetes","topology.kubernetes.io/region":"cn-hangzhou","topology.kubernetes.io/zone":"cn-hangzhou-b"},"CLUSTER_ID":"Kubernetes"},"watches":{"type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment":["outbound|15014||istiod.istio-system.svc.cluster.local","outbound|15010||istiod.istio-system.svc.cluster.local","outbound|53||kube-dns.kube-system.svc.cluster.local","outbound|11280||storage-monitor-service.kube-system.svc.cluster.local","outbound|8848||nacos.default.svc.cluster.local","outbound|20000||dubbo-go-app.default.svc.cluster.local","outbound|9153||kube-dns.kube-system.svc.cluster.local","","outbound|443||storage-crd-validate-service.kube-system.svc.cluster.local","outbound|443||metrics-server.kube-system.svc.cluster.local","outbound|15012||istiod.istio-system.svc.cluster.local","outbound|443||istiod.istio-system.svc.cluster.local","outbound|80||heapster.kube-system.svc.cluster.local","outbound|20000||laurence-svc.default.svc.cluster.local","outbound|443||kubernetes.default.svc.cluster.local"]}}]}` + key1 = "providers:api.Greeter::" + val1 = "dubbo-go-app.default.svc.cluster.local:20000" + key2 = "providers:grpc.reflection.v1alpha.ServerReflection::" + val2 = "dubbo-go-app.default.svc.cluster.local:20000" +) + +func TestADSZResponseGetMap(t *testing.T) { + adszRsp := &ADSZResponse{} + assert.Nil(t, json.Unmarshal([]byte(debugAdszDataFoo), adszRsp)) + + adszMap := adszRsp.GetMap() + fmt.Println(adszMap) + assert.True(t, len(adszMap) == 2) + v1, ok1 := adszMap[key1] + assert.True(t, ok1) + assert.Equal(t, val1, v1) + + v2, ok2 := adszMap[key2] + assert.True(t, ok2) + assert.Equal(t, val2, v2) +} diff --git a/remoting/xds/interfaceMapping/handler.go b/remoting/xds/interfaceMapping/handler.go new file mode 100644 index 0000000000..a92e6d149d --- /dev/null +++ b/remoting/xds/interfaceMapping/handler.go @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package interfaceMapping + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "sync" + "time" +) + +import ( + structpb "github.com/golang/protobuf/ptypes/struct" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common/logger" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/xds/client" +) + +const ( + authorizationHeader = "Authorization" + istiodTokenPrefix = "Bearer " +) + +type InterfaceMapHandlerImpl struct { + hostAddr common.Addr + + istioDebugAddr common.Addr + + xdsClient client.XDSClient + + istioTokenPath string + + /* + interfaceAppNameMap store map of serviceUniqueKey -> hostAddr + */ + interfaceAppNameMap map[string]string + interfaceAppNameMapLock sync.RWMutex + + /* + interfaceNameHostAddrMap cache the dubbo interface unique key -> hostName + the data is read from istiod:8080/debug/adsz, connection metadata["LABELS"]["DUBBO_GO"] + */ + interfaceNameHostAddrMap map[string]string + interfaceNameHostAddrMapLock sync.RWMutex +} + +func (i *InterfaceMapHandlerImpl) UnRegister(serviceUniqueKey string) error { + i.interfaceAppNameMapLock.Lock() + delete(i.interfaceAppNameMap, serviceUniqueKey) + i.interfaceAppNameMapLock.Unlock() + return i.xdsClient.SetMetadata(i.interfaceAppNameMap2DubboGoMetadata()) +} + +func (i *InterfaceMapHandlerImpl) Register(serviceUniqueKey string) error { + i.interfaceAppNameMapLock.Lock() + i.interfaceAppNameMap[serviceUniqueKey] = i.hostAddr.String() + i.interfaceAppNameMapLock.Unlock() + return i.xdsClient.SetMetadata(i.interfaceAppNameMap2DubboGoMetadata()) +} + +func (i *InterfaceMapHandlerImpl) GetHostAddrMap(serviceUniqueKey string) (string, error) { + i.interfaceNameHostAddrMapLock.RLock() + if hostAddr, ok := i.interfaceNameHostAddrMap[serviceUniqueKey]; ok { + return hostAddr, nil + } + i.interfaceNameHostAddrMapLock.RUnlock() + + for { + if interfaceHostAddrMap, err := i.getServiceUniqueKeyHostAddrMapFromPilot(); err != nil { + return "", err + } else { + i.interfaceNameHostAddrMapLock.Lock() + i.interfaceNameHostAddrMap = interfaceHostAddrMap + i.interfaceNameHostAddrMapLock.Unlock() + hostName, ok := interfaceHostAddrMap[serviceUniqueKey] + if !ok { + logger.Infof("[XDS Wrapped Client] Try getting interface %s 's host from istio %d:8080\n", serviceUniqueKey, i.istioDebugAddr) + time.Sleep(time.Millisecond * 100) + continue + } + return hostName, nil + } + } +} + +// getServiceUniqueKeyHostAddrMapFromPilot get map of service key like 'provider::api.Greeter' to host addr like +// 'dubbo-go-app.default.svc.cluster.local:20000' +func (i *InterfaceMapHandlerImpl) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { + req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/debug/adsz", i.istioDebugAddr.String()), nil) + token, err := os.ReadFile(i.istioTokenPath) + if err != nil { + return nil, err + } + req.Header.Add(authorizationHeader, istiodTokenPrefix+string(token)) + rsp, err := http.DefaultClient.Do(req) + if err != nil { + logger.Infof("[XDS Wrapped Client] Try getting interface host map from istio IP %s with error %s\n", + i.istioDebugAddr, err) + return nil, err + } + + data, err := ioutil.ReadAll(rsp.Body) + if err != nil { + return nil, err + } + adszRsp := &ADSZResponse{} + if err := json.Unmarshal(data, adszRsp); err != nil { + return nil, err + } + return adszRsp.GetMap(), nil +} + +func (i *InterfaceMapHandlerImpl) interfaceAppNameMap2DubboGoMetadata() *structpb.Struct { + i.interfaceAppNameMapLock.RLock() + defer i.interfaceAppNameMapLock.RUnlock() + data, _ := json.Marshal(i.interfaceAppNameMap) + return GetDubboGoMetadata(string(data)) +} + +func NewInterfaceMapHandlerImpl(xdsClient client.XDSClient, istioTokenPath string, istioDebugAddr, hostAddr common.Addr) InterfaceMapHandler { + return &InterfaceMapHandlerImpl{ + xdsClient: xdsClient, + interfaceAppNameMap: map[string]string{}, + interfaceNameHostAddrMap: map[string]string{}, + istioDebugAddr: istioDebugAddr, + hostAddr: hostAddr, + istioTokenPath: istioTokenPath, + } +} + +type InterfaceMapHandler interface { + Register(string) error + UnRegister(string) error + GetHostAddrMap(string) (string, error) +} diff --git a/remoting/xds/interfaceMapping/handler_test.go b/remoting/xds/interfaceMapping/handler_test.go new file mode 100644 index 0000000000..feca88dcb1 --- /dev/null +++ b/remoting/xds/interfaceMapping/handler_test.go @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package interfaceMapping + +import ( + "net/http" + "os" + "testing" + "time" +) + +import ( + structpb "github.com/golang/protobuf/ptypes/struct" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common/constant" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/xds/client/mocks" +) + +const ( + istioDebugPortFoo = "38080" + istiodDebugAddrStrFoo = "127.0.0.1:" + istioDebugPortFoo + localPodServiceAddr = "dubbo-go-app.default.svc.cluster.local:20000" + serviceKey1 = "providers:api.Greeter::" + serviceKey2 = "providers:grpc.reflection.v1alpha.ServerReflection::" + metadataService1 = `{"` + serviceKey1 + `":"` + localPodServiceAddr + `"}` + metadataService1And2 = `{"` + serviceKey1 + `":"` + localPodServiceAddr + `","` + serviceKey2 + `":"` + localPodServiceAddr + `"}` + metadataService2 = `{"` + serviceKey2 + `":"` + localPodServiceAddr + `"}` + metadataNoService = `{}` + istioTokenPathFoo = "/tmp/dubbo-go-mesh-test-token" + istioTokenFoo = "mock-token" +) + +func TestNewInterfaceMapHandler(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + assert.NotNil(t, interfaceMapHandler) +} + +func TestInterfaceMapHandlerRegisterAndUnregister(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + mockXDSClient.On("SetMetadata", mock.AnythingOfType("*structpb.Struct")).Return(nil) + + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + + assert.Nil(t, interfaceMapHandler.Register(serviceKey1)) + assert.Nil(t, interfaceMapHandler.Register(serviceKey2)) + assert.Nil(t, interfaceMapHandler.UnRegister(serviceKey1)) + assert.Nil(t, interfaceMapHandler.UnRegister(serviceKey2)) + + mockXDSClient.AssertCalled(t, "SetMetadata", mock.MatchedBy(getMatchFunction(metadataService1))) + mockXDSClient.AssertCalled(t, "SetMetadata", mock.MatchedBy(getMatchFunction(metadataService1And2))) + mockXDSClient.AssertCalled(t, "SetMetadata", mock.MatchedBy(getMatchFunction(metadataService2))) + mockXDSClient.AssertCalled(t, "SetMetadata", mock.MatchedBy(getMatchFunction(metadataNoService))) +} + +func TestGetServiceUniqueKeyHostAddrMapFromPilot(t *testing.T) { + mockXDSClient := &mocks.XDSClient{} + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + assert.Nil(t, generateMockToken()) + + // 1. start mock http server + http.HandleFunc("/debug/adsz", func(writer http.ResponseWriter, request *http.Request) { + if len(request.Header[authorizationHeader]) != 1 { + writer.Write([]byte("")) + return + } + if request.Header[authorizationHeader][0] != istiodTokenPrefix+istioTokenFoo { + writer.Write([]byte("")) + return + } + writer.Write([]byte(debugAdszDataFoo)) + }) + go http.ListenAndServe(":"+istioDebugPortFoo, nil) + time.Sleep(time.Second) + + // 2. get map from client + hostAddr1, err := interfaceMapHandler.GetHostAddrMap(serviceKey1) + assert.Nil(t, err) + assert.Equal(t, localPodServiceAddr, hostAddr1) + + // 3. assert + hostAddr2, err := interfaceMapHandler.GetHostAddrMap(serviceKey2) + assert.Nil(t, err) + assert.Equal(t, localPodServiceAddr, hostAddr2) + +} + +func getMatchFunction(metadata string) func(abc *structpb.Struct) bool { + return func(abc *structpb.Struct) bool { + metadatas, ok := abc.Fields[constant.XDSMetadataLabelsKey] + if !ok { + return false + } + subFields := metadatas.GetStructValue().GetFields() + subValue, ok2 := subFields[constant.XDSMetadataDubboGoMapperKey] + if !ok2 { + return false + } + if subValue.GetStringValue() != metadata { + return false + } + return true + } +} + +func generateMockToken() error { + return os.WriteFile(istioTokenPathFoo, []byte(istioTokenFoo), 0777) +} diff --git a/remoting/xds/interfaceMapping/metadata.go b/remoting/xds/interfaceMapping/metadata.go new file mode 100644 index 0000000000..9a44ddd5c0 --- /dev/null +++ b/remoting/xds/interfaceMapping/metadata.go @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package interfaceMapping + +import ( + structpb "github.com/golang/protobuf/ptypes/struct" +) + +import ( + "dubbo.apache.org/dubbo-go/v3/common/constant" +) + +// GetDubboGoMetadata create metadata of dubbo-go-app and register to istiod +func GetDubboGoMetadata(dubboGoMetadata string) *structpb.Struct { + return &structpb.Struct{ + Fields: map[string]*structpb.Value{ + constant.XDSMetadataClusterIDKey: { + // Set cluster id to Kubernetes to ensure dubbo-go's xds client can get service + // istiod.istio-system.svc.cluster.local's + // pods ip from istiod by eds, to call no-endpoint port of istio like 8080 + Kind: &structpb.Value_StringValue{StringValue: constant.XDSMetadataDefaultDomainName}, + }, + constant.XDSMetadataLabelsKey: { + Kind: &structpb.Value_StructValue{StructValue: &structpb.Struct{ + Fields: map[string]*structpb.Value{ + constant.XDSMetadataDubboGoMapperKey: { + Kind: &structpb.Value_StringValue{StringValue: dubboGoMetadata}, + }, + }, + }}, + }, + }, + } +} diff --git a/remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go b/remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go new file mode 100644 index 0000000000..fd22475501 --- /dev/null +++ b/remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by mockery v2.9.4. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" +) + +// InterfaceMapHandler is an autogenerated mock type for the InterfaceMapHandler type +type InterfaceMapHandler struct { + mock.Mock +} + +// GetHostAddrMap provides a mock function with given fields: _a0 +func (_m *InterfaceMapHandler) GetHostAddrMap(_a0 string) (string, error) { + ret := _m.Called(_a0) + + var r0 string + if rf, ok := ret.Get(0).(func(string) string); ok { + r0 = rf(_a0) + } else { + r0 = ret.Get(0).(string) + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// Register provides a mock function with given fields: _a0 +func (_m *InterfaceMapHandler) Register(_a0 string) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// UnRegister provides a mock function with given fields: _a0 +func (_m *InterfaceMapHandler) UnRegister(_a0 string) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go new file mode 100644 index 0000000000..e61c8d5efe --- /dev/null +++ b/remoting/xds/xds_client_factory.go @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xds + +import ( + v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +import ( + xdsCommon "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/interfaceMapping" + "dubbo.apache.org/dubbo-go/v3/xds/client" + "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" +) + +// xdsClientFactoryFunction generates new xds client +// when running ut, it's for for ut to replace +var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr xdsCommon.Addr) (client.XDSClient, error) { + // todo fix these ugly magic num + v3NodeProto := &v3corepb.Node{ + Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", + UserAgentName: gRPCUserAgentName, + UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, + ClientFeatures: []string{clientFeatureNoOverprovisioning}, + Metadata: interfaceMapping.GetDubboGoMetadata(""), + } + + nonNilCredsConfigV2 := &bootstrap.Config{ + XDSServer: &bootstrap.ServerConfig{ + ServerURI: istioAddr.String(), + Creds: grpc.WithTransportCredentials(insecure.NewCredentials()), + TransportAPI: version.TransportV3, + NodeProto: v3NodeProto, + }, + ClientDefaultListenerResourceNameTemplate: "%s", + } + + newClient, err := client.NewWithConfig(nonNilCredsConfigV2) + if err != nil { + return nil, err + } + return newClient, nil +} diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go index b3e02650e6..367e51f720 100644 --- a/xds/balancer/balancer.go +++ b/xds/balancer/balancer.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package balancer installs all the xds balancers. package balancer diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index f4cf31fa9e..d2e71d2b76 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package cdsbalancer implements a balancer to handle CDS responses. package cdsbalancer diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go index 9e3d8c67ac..76e657fe9d 100644 --- a/xds/balancer/cdsbalancer/cluster_handler.go +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package cdsbalancer import ( diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go index 547b4d9faa..6dd48b9f65 100644 --- a/xds/balancer/cdsbalancer/logging.go +++ b/xds/balancer/cdsbalancer/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package cdsbalancer import ( diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go index b93266fe57..536c6c756e 100644 --- a/xds/balancer/clusterimpl/clusterimpl.go +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package clusterimpl implements the xds_cluster_impl balancing policy. It // handles the cluster features (e.g. circuit_breaking, RPC dropping). // diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go index 59aba156db..ad50044108 100644 --- a/xds/balancer/clusterimpl/config.go +++ b/xds/balancer/clusterimpl/config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterimpl import ( diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go index baf0c18f03..c8c4b6e279 100644 --- a/xds/balancer/clusterimpl/config_test.go +++ b/xds/balancer/clusterimpl/config_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterimpl import ( diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go index 790a50676e..3dd733ac63 100644 --- a/xds/balancer/clusterimpl/logging.go +++ b/xds/balancer/clusterimpl/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterimpl import ( diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go index 183d852b59..001567cb5e 100644 --- a/xds/balancer/clusterimpl/picker.go +++ b/xds/balancer/clusterimpl/picker.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterimpl import ( diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go index 07eb81e56c..9712eaa0e4 100644 --- a/xds/balancer/clustermanager/balancerstateaggregator.go +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clustermanager import ( diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go index 3dfc10a6fd..d14d2b2a68 100644 --- a/xds/balancer/clustermanager/clustermanager.go +++ b/xds/balancer/clustermanager/clustermanager.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package clustermanager implements the cluster manager LB policy for xds. package clustermanager diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go index 9121a328f3..15cd28cbfa 100644 --- a/xds/balancer/clustermanager/config.go +++ b/xds/balancer/clustermanager/config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clustermanager import ( diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go index fcec6d6b90..5d6b41b4a4 100644 --- a/xds/balancer/clustermanager/picker.go +++ b/xds/balancer/clustermanager/picker.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clustermanager import ( diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go index 0185907b67..6e3885ab07 100644 --- a/xds/balancer/clusterresolver/clusterresolver.go +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package clusterresolver contains EDS balancer implementation. package clusterresolver diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go index a70d32d389..feb322af88 100644 --- a/xds/balancer/clusterresolver/config.go +++ b/xds/balancer/clusterresolver/config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go index 8ca82c8a98..8394e6adc4 100644 --- a/xds/balancer/clusterresolver/configbuilder.go +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go index e1a7d3e2f7..35540826c7 100644 --- a/xds/balancer/clusterresolver/logging.go +++ b/xds/balancer/clusterresolver/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go index 766e327df0..0b0fa3c26f 100644 --- a/xds/balancer/clusterresolver/resource_resolver.go +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go index 5315ae3830..33e21894c7 100644 --- a/xds/balancer/clusterresolver/resource_resolver_dns.go +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go index f3b5d4562c..b3c4cad7a1 100644 --- a/xds/balancer/clusterresolver/weightedtarget_config.go +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package clusterresolver import ( diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go index 8fb977a526..992118fcf5 100644 --- a/xds/balancer/loadstore/load_store_wrapper.go +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package loadstore contains the loadStoreWrapper shared by the balancers. package loadstore diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go index 7553269704..930e8e7588 100644 --- a/xds/balancer/orca/orca.go +++ b/xds/balancer/orca/orca.go @@ -14,6 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/* + * Copyright gRPC authors. + */ // Package orca implements Open Request Cost Aggregation. package orca diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go index 19405dc180..9386bc027c 100644 --- a/xds/balancer/priority/balancer.go +++ b/xds/balancer/priority/balancer.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package priority implements the priority balancer. // // This balancer will be kept in internal until we use it in the xds balancers, diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go index fbaf38dc98..7517999eeb 100644 --- a/xds/balancer/priority/balancer_child.go +++ b/xds/balancer/priority/balancer_child.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go index 3e4166dec8..aa966fc304 100644 --- a/xds/balancer/priority/balancer_priority.go +++ b/xds/balancer/priority/balancer_priority.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go index f9c236e321..2f79221f1f 100644 --- a/xds/balancer/priority/config.go +++ b/xds/balancer/priority/config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go index 2cdbd90f46..165f7a5cd5 100644 --- a/xds/balancer/priority/config_test.go +++ b/xds/balancer/priority/config_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go index 24c9e2db13..07838660b6 100644 --- a/xds/balancer/priority/ignore_resolve_now.go +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go index 392992969b..a4d870cecf 100644 --- a/xds/balancer/priority/logging.go +++ b/xds/balancer/priority/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/priority/utils.go b/xds/balancer/priority/utils.go index 7724d77391..ea2956d6f8 100644 --- a/xds/balancer/priority/utils.go +++ b/xds/balancer/priority/utils.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority func equalStringSlice(a, b []string) bool { diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go index 3bea5419b8..027222b869 100644 --- a/xds/balancer/priority/utils_test.go +++ b/xds/balancer/priority/utils_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package priority import ( diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go index 475e83ec26..ec94fe89cb 100644 --- a/xds/balancer/ringhash/config.go +++ b/xds/balancer/ringhash/config.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go index 075623bf72..c9f8fc7fed 100644 --- a/xds/balancer/ringhash/config_test.go +++ b/xds/balancer/ringhash/config_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go index eb7457a46f..2265075f79 100644 --- a/xds/balancer/ringhash/logging.go +++ b/xds/balancer/ringhash/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go index 39314b9689..59f93123fc 100644 --- a/xds/balancer/ringhash/picker.go +++ b/xds/balancer/ringhash/picker.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go index f1dba7d849..cb5d861ba0 100644 --- a/xds/balancer/ringhash/ring.go +++ b/xds/balancer/ringhash/ring.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go index 2dbb7b5610..e9d967a4cd 100644 --- a/xds/balancer/ringhash/ring_test.go +++ b/xds/balancer/ringhash/ring_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go index 1f5f3156b6..cfc7101bdc 100644 --- a/xds/balancer/ringhash/ringhash.go +++ b/xds/balancer/ringhash/ringhash.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package ringhash implements the ringhash balancer. package ringhash diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go index 3bf0ad81e4..2db9d21c93 100644 --- a/xds/balancer/ringhash/util.go +++ b/xds/balancer/ringhash/util.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package ringhash import ( diff --git a/xds/client/attributes.go b/xds/client/attributes.go index 9bad8501f8..97d536a196 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/authority.go b/xds/client/authority.go index 35659993e7..50abb515e1 100644 --- a/xds/client/authority.go +++ b/xds/client/authority.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index e7d547a5e7..2ce5410a13 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package bootstrap provides the functionality to initialize certain aspects // of an xDS client by reading a bootstrap file. package bootstrap diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 551b135bf2..2d9ce1220e 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package bootstrap import ( diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go index 5ad007c96f..adeb240b3f 100644 --- a/xds/client/bootstrap/logging.go +++ b/xds/client/bootstrap/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package bootstrap import ( diff --git a/xds/client/bootstrap/template.go b/xds/client/bootstrap/template.go index 58e8e9b708..e88fec4410 100644 --- a/xds/client/bootstrap/template.go +++ b/xds/client/bootstrap/template.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package bootstrap import ( diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go index ca3251d2ce..cdb125a1d5 100644 --- a/xds/client/bootstrap/template_test.go +++ b/xds/client/bootstrap/template_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package bootstrap import ( diff --git a/xds/client/client.go b/xds/client/client.go index d65dc4f250..9aff7e7df9 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // package client implements a full fledged gRPC client for the xDS API used // by the xds resolver and balancer implementations. package client diff --git a/xds/client/controller.go b/xds/client/controller.go index 12556d016d..509079664f 100644 --- a/xds/client/controller.go +++ b/xds/client/controller.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go index 4bf2fa6104..6fc6b75b56 100644 --- a/xds/client/controller/controller.go +++ b/xds/client/controller/controller.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package controller contains implementation to connect to the control plane. // Including starting the ClientConn, starting the xDS stream, and // sending/receiving messages. diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go index 3b9e55041f..25c0845411 100644 --- a/xds/client/controller/loadreport.go +++ b/xds/client/controller/loadreport.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package controller import ( diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go index aafbda85ec..5a5ce315c7 100644 --- a/xds/client/controller/transport.go +++ b/xds/client/controller/transport.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package controller import ( diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go index 36c6d24818..b970c85c27 100644 --- a/xds/client/controller/version/v2/client.go +++ b/xds/client/controller/version/v2/client.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package v2 provides xDS v2 transport protocol specific functionality. package v2 diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go index 55763e236c..37f44bd429 100644 --- a/xds/client/controller/version/v2/loadreport.go +++ b/xds/client/controller/version/v2/loadreport.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package v2 import ( diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go index a7bcf505a4..ce68cc1216 100644 --- a/xds/client/controller/version/v3/client.go +++ b/xds/client/controller/version/v3/client.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package v3 provides xDS v3 transport protocol specific functionality. package v3 diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go index 5a778eac6d..63824539ad 100644 --- a/xds/client/controller/version/v3/loadreport.go +++ b/xds/client/controller/version/v3/loadreport.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package v3 import ( diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go index d45ec17ca2..62c7fb9c5c 100644 --- a/xds/client/controller/version/version.go +++ b/xds/client/controller/version/version.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package version defines APIs to deal with different versions of xDS. package version diff --git a/xds/client/dump.go b/xds/client/dump.go index 51d386623a..1425dd371e 100644 --- a/xds/client/dump.go +++ b/xds/client/dump.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/load/reporter.go b/xds/client/load/reporter.go index a0ba224f24..446206a5fb 100644 --- a/xds/client/load/reporter.go +++ b/xds/client/load/reporter.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package load // PerClusterReporter wraps the methods from the loadStore that are used here. diff --git a/xds/client/load/store.go b/xds/client/load/store.go index 14fc5308f5..9aea03a5cf 100644 --- a/xds/client/load/store.go +++ b/xds/client/load/store.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package load provides functionality to record and maintain load data. package load diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go index be4b77e7b4..9148e82da6 100644 --- a/xds/client/load/store_test.go +++ b/xds/client/load/store_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package load import ( diff --git a/xds/client/loadreport.go b/xds/client/loadreport.go index 97695038bc..ff02fb9a66 100644 --- a/xds/client/loadreport.go +++ b/xds/client/loadreport.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/logging.go b/xds/client/logging.go index 6949dd77f3..808d83712b 100644 --- a/xds/client/logging.go +++ b/xds/client/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/mocks/XDSClient.go b/xds/client/mocks/XDSClient.go new file mode 100644 index 0000000000..395fe4ef58 --- /dev/null +++ b/xds/client/mocks/XDSClient.go @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by mockery v2.9.4. DO NOT EDIT. + +package mocks + +import ( + mock "github.com/stretchr/testify/mock" + + structpb "google.golang.org/protobuf/types/known/structpb" +) + +import ( + bootstrap "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" + load "dubbo.apache.org/dubbo-go/v3/xds/client/load" + resource "dubbo.apache.org/dubbo-go/v3/xds/client/resource" +) + +// XDSClient is an autogenerated mock type for the XDSClient type +type XDSClient struct { + mock.Mock +} + +// BootstrapConfig provides a mock function with given fields: +func (_m *XDSClient) BootstrapConfig() *bootstrap.Config { + ret := _m.Called() + + var r0 *bootstrap.Config + if rf, ok := ret.Get(0).(func() *bootstrap.Config); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*bootstrap.Config) + } + } + + return r0 +} + +// Close provides a mock function with given fields: +func (_m *XDSClient) Close() { + _m.Called() +} + +// DumpCDS provides a mock function with given fields: +func (_m *XDSClient) DumpCDS() map[string]resource.UpdateWithMD { + ret := _m.Called() + + var r0 map[string]resource.UpdateWithMD + if rf, ok := ret.Get(0).(func() map[string]resource.UpdateWithMD); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]resource.UpdateWithMD) + } + } + + return r0 +} + +// DumpEDS provides a mock function with given fields: +func (_m *XDSClient) DumpEDS() map[string]resource.UpdateWithMD { + ret := _m.Called() + + var r0 map[string]resource.UpdateWithMD + if rf, ok := ret.Get(0).(func() map[string]resource.UpdateWithMD); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]resource.UpdateWithMD) + } + } + + return r0 +} + +// DumpLDS provides a mock function with given fields: +func (_m *XDSClient) DumpLDS() map[string]resource.UpdateWithMD { + ret := _m.Called() + + var r0 map[string]resource.UpdateWithMD + if rf, ok := ret.Get(0).(func() map[string]resource.UpdateWithMD); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]resource.UpdateWithMD) + } + } + + return r0 +} + +// DumpRDS provides a mock function with given fields: +func (_m *XDSClient) DumpRDS() map[string]resource.UpdateWithMD { + ret := _m.Called() + + var r0 map[string]resource.UpdateWithMD + if rf, ok := ret.Get(0).(func() map[string]resource.UpdateWithMD); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]resource.UpdateWithMD) + } + } + + return r0 +} + +// ReportLoad provides a mock function with given fields: server +func (_m *XDSClient) ReportLoad(server string) (*load.Store, func()) { + ret := _m.Called(server) + + var r0 *load.Store + if rf, ok := ret.Get(0).(func(string) *load.Store); ok { + r0 = rf(server) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*load.Store) + } + } + + var r1 func() + if rf, ok := ret.Get(1).(func(string) func()); ok { + r1 = rf(server) + } else { + if ret.Get(1) != nil { + r1 = ret.Get(1).(func()) + } + } + + return r0, r1 +} + +// SetMetadata provides a mock function with given fields: _a0 +func (_m *XDSClient) SetMetadata(_a0 *structpb.Struct) error { + ret := _m.Called(_a0) + + var r0 error + if rf, ok := ret.Get(0).(func(*structpb.Struct) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// WatchCluster provides a mock function with given fields: _a0, _a1 +func (_m *XDSClient) WatchCluster(_a0 string, _a1 func(resource.ClusterUpdate, error)) func() { + ret := _m.Called(_a0, _a1) + + var r0 func() + if rf, ok := ret.Get(0).(func(string, func(resource.ClusterUpdate, error)) func()); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func()) + } + } + + return r0 +} + +// WatchEndpoints provides a mock function with given fields: clusterName, edsCb +func (_m *XDSClient) WatchEndpoints(clusterName string, edsCb func(resource.EndpointsUpdate, error)) func() { + ret := _m.Called(clusterName, edsCb) + + var r0 func() + if rf, ok := ret.Get(0).(func(string, func(resource.EndpointsUpdate, error)) func()); ok { + r0 = rf(clusterName, edsCb) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func()) + } + } + + return r0 +} + +// WatchListener provides a mock function with given fields: _a0, _a1 +func (_m *XDSClient) WatchListener(_a0 string, _a1 func(resource.ListenerUpdate, error)) func() { + ret := _m.Called(_a0, _a1) + + var r0 func() + if rf, ok := ret.Get(0).(func(string, func(resource.ListenerUpdate, error)) func()); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func()) + } + } + + return r0 +} + +// WatchRouteConfig provides a mock function with given fields: _a0, _a1 +func (_m *XDSClient) WatchRouteConfig(_a0 string, _a1 func(resource.RouteConfigUpdate, error)) func() { + ret := _m.Called(_a0, _a1) + + var r0 func() + if rf, ok := ret.Get(0).(func(string, func(resource.RouteConfigUpdate, error)) func()); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(func()) + } + } + + return r0 +} diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go index d4895cd4c0..d7f80667d3 100644 --- a/xds/client/pubsub/dump.go +++ b/xds/client/pubsub/dump.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package pubsub import ( diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go index be1be057a8..cf29dd018d 100644 --- a/xds/client/pubsub/interface.go +++ b/xds/client/pubsub/interface.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package pubsub import ( diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go index 32385118d7..678b9ee981 100644 --- a/xds/client/pubsub/pubsub.go +++ b/xds/client/pubsub/pubsub.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package pubsub implements a utility type to maintain resource watchers and // the updates. // diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 1f0787426d..6b546ccc49 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package pubsub import ( diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go index 6779c4fdd9..5aec6757e7 100644 --- a/xds/client/pubsub/watch.go +++ b/xds/client/pubsub/watch.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package pubsub import ( diff --git a/xds/client/requests_counter.go b/xds/client/requests_counter.go index 0d93d10327..34d0cb4bd8 100644 --- a/xds/client/requests_counter.go +++ b/xds/client/requests_counter.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go index 82b51520da..5b6ce7bbea 100644 --- a/xds/client/resource/errors.go +++ b/xds/client/resource/errors.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go index 50342ecb9e..6a77e8db87 100644 --- a/xds/client/resource/filter_chain.go +++ b/xds/client/resource/filter_chain.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go index e4ecd4a93c..d26dcedd58 100644 --- a/xds/client/resource/locality_id.go +++ b/xds/client/resource/locality_id.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package internal contains functions/structs shared by xds // balancers/resolvers. package resource diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go index 0605634aa0..6d203895d0 100644 --- a/xds/client/resource/matcher.go +++ b/xds/client/resource/matcher.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go index f2565e97fe..859b142cdf 100644 --- a/xds/client/resource/matcher_path.go +++ b/xds/client/resource/matcher_path.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go index 1aaaad31a2..f8959ac7cc 100644 --- a/xds/client/resource/name.go +++ b/xds/client/resource/name.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go index c4be177987..c71b0c233b 100644 --- a/xds/client/resource/type.go +++ b/xds/client/resource/type.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go index 16bef47d5f..1c6b42ba20 100644 --- a/xds/client/resource/type_cds.go +++ b/xds/client/resource/type_cds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/type_eds.go b/xds/client/resource/type_eds.go index 88b5c09594..499e9491b0 100644 --- a/xds/client/resource/type_eds.go +++ b/xds/client/resource/type_eds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go index cd5403b0fc..490c804924 100644 --- a/xds/client/resource/type_lds.go +++ b/xds/client/resource/type_lds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go index 999d9e6c68..5312cfb49a 100644 --- a/xds/client/resource/type_rds.go +++ b/xds/client/resource/type_rds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go index c02d3d7b30..7252df0d4c 100644 --- a/xds/client/resource/unmarshal.go +++ b/xds/client/resource/unmarshal.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package resource contains functions to proto xds updates (unmarshal from // proto), and types for the resource updates. package resource diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go index 8af5af0650..015104eb0a 100644 --- a/xds/client/resource/unmarshal_cds.go +++ b/xds/client/resource/unmarshal_cds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go index 3d0bfdf29f..1f3921f2e8 100644 --- a/xds/client/resource/unmarshal_eds.go +++ b/xds/client/resource/unmarshal_eds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index ddda349c5a..624a72ef40 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go index af44b32cab..f155cbee8f 100644 --- a/xds/client/resource/unmarshal_rds.go +++ b/xds/client/resource/unmarshal_rds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resource import ( @@ -177,7 +181,8 @@ func generateRetryConfig(rp *v3routepb.RetryPolicy) (*RetryConfig, error) { cfg := &RetryConfig{RetryOn: make(map[codes.Code]bool)} for _, s := range strings.Split(rp.GetRetryOn(), ",") { switch strings.TrimSpace(strings.ToLower(s)) { - case "canceled": + // FIXME, is this misspelled by grpc? + case "cancel" + "led": cfg.RetryOn[codes.Canceled] = true case "deadline-exceeded": cfg.RetryOn[codes.DeadlineExceeded] = true diff --git a/xds/client/resource/version/version.go b/xds/client/resource/version/version.go index 6fb1b3a22a..93c7127e5e 100644 --- a/xds/client/resource/version/version.go +++ b/xds/client/resource/version/version.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package version defines constants to distinguish between supported xDS API // versions. package version diff --git a/xds/client/singleton.go b/xds/client/singleton.go index c42877e00e..8a8c17b654 100644 --- a/xds/client/singleton.go +++ b/xds/client/singleton.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/client/watchers.go b/xds/client/watchers.go index acae4d1b2c..cd9c3d2954 100644 --- a/xds/client/watchers.go +++ b/xds/client/watchers.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package client import ( diff --git a/xds/clusterspecifier/cluster_specifier.go b/xds/clusterspecifier/cluster_specifier.go index 43a994bc4b..6afca4083d 100644 --- a/xds/clusterspecifier/cluster_specifier.go +++ b/xds/clusterspecifier/cluster_specifier.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package clusterspecifier contains the ClusterSpecifier interface and a registry for // storing and retrieving their implementations. package clusterspecifier diff --git a/xds/csds/csds.go b/xds/csds/csds.go index 0182584d80..818e13c01b 100644 --- a/xds/csds/csds.go +++ b/xds/csds/csds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package csds implements features to dump the status (xDS responses) the // xds_client is using. // diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go index a935e87226..04b45e5096 100644 --- a/xds/httpfilter/fault/fault.go +++ b/xds/httpfilter/fault/fault.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package fault implements the Envoy Fault Injection HTTP filter. package fault diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go index d022ff3728..a82794b6aa 100644 --- a/xds/httpfilter/httpfilter.go +++ b/xds/httpfilter/httpfilter.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package httpfilter contains the HTTPFilter interface and a registry for // storing and retrieving their implementations. package httpfilter diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go index d022b86244..37b2aa7e21 100644 --- a/xds/httpfilter/rbac/rbac.go +++ b/xds/httpfilter/rbac/rbac.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package rbac implements the Envoy RBAC HTTP filter. package rbac diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go index 3fbdec00d3..41d4f05e8b 100644 --- a/xds/httpfilter/router/router.go +++ b/xds/httpfilter/router/router.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package router implements the Envoy Router HTTP filter. package router diff --git a/xds/internal/internal.go b/xds/internal/internal.go index 2ba3d47c84..0da8c9d068 100644 --- a/xds/internal/internal.go +++ b/xds/internal/internal.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package internal contains gRPC-internal code, to avoid polluting // the godoc of the top-level grpc package. It must not import any grpc // symbols to avoid circular dependencies. diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go index 04544ee10a..f42a482849 100644 --- a/xds/resolver/logging.go +++ b/xds/resolver/logging.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resolver import ( diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go index ef44f757ee..3f0a6c797b 100644 --- a/xds/resolver/serviceconfig.go +++ b/xds/resolver/serviceconfig.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resolver import ( diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go index 3f9e4f98a4..e407b3a0e6 100644 --- a/xds/resolver/watch_service.go +++ b/xds/resolver/watch_service.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package resolver import ( diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go index 436eabd5c8..fa22475e10 100644 --- a/xds/resolver/xds_resolver.go +++ b/xds/resolver/xds_resolver.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package resolver implements the xds resolver, that does LDS and RDS to find // the cluster to use. package resolver diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index d3bead7b52..ed9b0dd1ea 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package server import ( diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go index fe474eeb49..db4a214ac6 100644 --- a/xds/server/listener_wrapper.go +++ b/xds/server/listener_wrapper.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package server contains internal server-side functionality used by the public // facing xds package. package server diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go index 6e6c8df8ef..f85cf0b200 100644 --- a/xds/server/rds_handler.go +++ b/xds/server/rds_handler.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package server import ( diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go index 538016d1b3..270b2c60c6 100644 --- a/xds/utils/backoff/backoff.go +++ b/xds/utils/backoff/backoff.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package backoff implement the backoff strategy for gRPC. // // This is kept in internal until the gRPC project decides whether or not to diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go index 993f4c18bb..bfd52209c1 100644 --- a/xds/utils/balancer/stub/stub.go +++ b/xds/utils/balancer/stub/stub.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package stub implements a balancer for testing purposes. package stub diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index 3c98c73547..f9fb9069f6 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -14,6 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/* + * Copyright gRPC authors. + */ // Package balancergroup implements a utility struct to bind multiple balancers // into one balancer. package balancergroup diff --git a/xds/utils/balancergroup/balancerstateaggregator.go b/xds/utils/balancergroup/balancerstateaggregator.go index a7eed1144a..6c3273fe70 100644 --- a/xds/utils/balancergroup/balancerstateaggregator.go +++ b/xds/utils/balancergroup/balancerstateaggregator.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package balancergroup import ( diff --git a/xds/utils/balancerload/load.go b/xds/utils/balancerload/load.go index e8c872d01f..baf10c41b9 100644 --- a/xds/utils/balancerload/load.go +++ b/xds/utils/balancerload/load.go @@ -14,6 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/* + * Copyright gRPC authors. + */ // Package balancerload defines APIs to parse server loads in trailers. The // parsed loads are sent to balancers in DoneInfo. package balancerload diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go index 5049fb0716..85b0de6262 100644 --- a/xds/utils/buffer/unbounded.go +++ b/xds/utils/buffer/unbounded.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package buffer provides an implementation of an unbounded buffer. package buffer diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index b84b9f9b24..12437db40d 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package xds contains non-user facing functionality of the xds credentials. package xds diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go index 5985e8464a..58893d44f5 100644 --- a/xds/utils/credentials/xds/handshake_info_test.go +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package xds import ( diff --git a/xds/utils/envconfig/envconfig.go b/xds/utils/envconfig/envconfig.go index 847348aae4..119062fa68 100644 --- a/xds/utils/envconfig/envconfig.go +++ b/xds/utils/envconfig/envconfig.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package envconfig contains grpc settings configured by environment variables. package envconfig diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index d0b5b83017..7f6c560258 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package envconfig import ( diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index 76d08cfb81..89bbdc775a 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package grpclog (internal) defines depth logging for grpc. package grpclog diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go index 5e277cec41..bbbcc786c3 100644 --- a/xds/utils/grpclog/prefixLogger.go +++ b/xds/utils/grpclog/prefixLogger.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpclog import ( diff --git a/xds/utils/grpcrand/grpcrand.go b/xds/utils/grpcrand/grpcrand.go index 8682bc3101..a026c890c2 100644 --- a/xds/utils/grpcrand/grpcrand.go +++ b/xds/utils/grpcrand/grpcrand.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package grpcrand implements math/rand functions in a concurrent-safe way // with a global random source, independent of math/rand's global source. package grpcrand diff --git a/xds/utils/grpcsync/event.go b/xds/utils/grpcsync/event.go index 9a19a0e758..69426ad0b2 100644 --- a/xds/utils/grpcsync/event.go +++ b/xds/utils/grpcsync/event.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package grpcsync implements additional synchronization primitives built upon // the sync package. package grpcsync diff --git a/xds/utils/grpcutil/encode_duration.go b/xds/utils/grpcutil/encode_duration.go index b7242e9a20..613a9f2ef3 100644 --- a/xds/utils/grpcutil/encode_duration.go +++ b/xds/utils/grpcutil/encode_duration.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/encode_duration_test.go b/xds/utils/grpcutil/encode_duration_test.go index d24e995855..c77c80139c 100644 --- a/xds/utils/grpcutil/encode_duration_test.go +++ b/xds/utils/grpcutil/encode_duration_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/grpcutil.go b/xds/utils/grpcutil/grpcutil.go index 432a3cfee5..211ee64a6b 100644 --- a/xds/utils/grpcutil/grpcutil.go +++ b/xds/utils/grpcutil/grpcutil.go @@ -15,5 +15,9 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package grpcutil provides utility functions used across the gRPC codebase. package grpcutil diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go index d39bcfda1e..87fe9d3d21 100644 --- a/xds/utils/grpcutil/metadata.go +++ b/xds/utils/grpcutil/metadata.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/method.go b/xds/utils/grpcutil/method.go index 00b587090e..369819ce83 100644 --- a/xds/utils/grpcutil/method.go +++ b/xds/utils/grpcutil/method.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/method_test.go b/xds/utils/grpcutil/method_test.go index 32491b4990..cfd0adc3e6 100644 --- a/xds/utils/grpcutil/method_test.go +++ b/xds/utils/grpcutil/method_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go index 35dd4e84bc..d12ae5f5ac 100644 --- a/xds/utils/grpcutil/regex.go +++ b/xds/utils/grpcutil/regex.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/grpcutil/regex_test.go b/xds/utils/grpcutil/regex_test.go index 6fc0c95466..8137f7cb0a 100644 --- a/xds/utils/grpcutil/regex_test.go +++ b/xds/utils/grpcutil/regex_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package grpcutil import ( diff --git a/xds/utils/hierarchy/hierarchy.go b/xds/utils/hierarchy/hierarchy.go index e31a267292..6bb8e7065f 100644 --- a/xds/utils/hierarchy/hierarchy.go +++ b/xds/utils/hierarchy/hierarchy.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package hierarchy contains functions to set and get hierarchy string from // addresses. // diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go index 5a8a317dbb..8629dd8082 100644 --- a/xds/utils/hierarchy/hierarchy_test.go +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package hierarchy import ( diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go index 795001a36c..cc3a94e56d 100644 --- a/xds/utils/matcher/matcher_header.go +++ b/xds/utils/matcher/matcher_header.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package matcher import ( diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go index d14cf9f44d..12f4465496 100644 --- a/xds/utils/matcher/matcher_header_test.go +++ b/xds/utils/matcher/matcher_header_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package matcher import ( diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go index fef02c7058..77ef637c27 100644 --- a/xds/utils/matcher/regex.go +++ b/xds/utils/matcher/regex.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package matcher import ( diff --git a/xds/utils/matcher/regex_test.go b/xds/utils/matcher/regex_test.go index 4c13b0a611..e5e3f9c077 100644 --- a/xds/utils/matcher/regex_test.go +++ b/xds/utils/matcher/regex_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package matcher import ( diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go index ab983846f0..7785af9839 100644 --- a/xds/utils/matcher/string_matcher.go +++ b/xds/utils/matcher/string_matcher.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package matcher contains types that need to be shared between code under // google.golang.org/grpc/xds/... and the rest of gRPC. package matcher diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go index 4ff5cba93a..96731adb8c 100644 --- a/xds/utils/matcher/string_matcher_test.go +++ b/xds/utils/matcher/string_matcher_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package matcher import ( diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go index 7c5b83d3f6..cbb1dd4f16 100644 --- a/xds/utils/metadata/metadata.go +++ b/xds/utils/metadata/metadata.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package metadata import ( diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go index 12384f479d..523cc444fb 100644 --- a/xds/utils/pretty/pretty.go +++ b/xds/utils/pretty/pretty.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package pretty defines helper functions to pretty-print structs for logging. package pretty diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go index 614bb689b8..0a330a1417 100644 --- a/xds/utils/rbac/matchers.go +++ b/xds/utils/rbac/matchers.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package rbac import ( diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go index c782c4b4d0..e8dd11cc11 100644 --- a/xds/utils/rbac/rbac_engine.go +++ b/xds/utils/rbac/rbac_engine.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package rbac provides service-level and method-level access control for a // service. See // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/rbac/v3/rbac.proto#role-based-access-control-rbac diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go index a5d3a0413e..8a47062f4c 100644 --- a/xds/utils/resolver/config_selector.go +++ b/xds/utils/resolver/config_selector.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package resolver provides internal resolver-related functionality. package resolver diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go index 93b3c52c2d..3447ab5459 100644 --- a/xds/utils/resolver/passthrough/passthrough.go +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package passthrough implements a pass-through resolver. It sends the target // name without scheme back to gRPC as resolved address. package passthrough diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go index 45c4f6527a..dbf9f18e9e 100644 --- a/xds/utils/resolver/unix/unix.go +++ b/xds/utils/resolver/unix/unix.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package unix implements a resolver for unix targets. package unix diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go index 9de85e9c5f..82cc7795bc 100644 --- a/xds/utils/serviceconfig/serviceconfig.go +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package serviceconfig contains utility functions to parse service config. package serviceconfig diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go index fe8de63cc8..ef34b3de02 100644 --- a/xds/utils/serviceconfig/serviceconfig_test.go +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package serviceconfig import ( diff --git a/xds/utils/transport/conn.go b/xds/utils/transport/conn.go index 80728cb517..a7884d7cd5 100644 --- a/xds/utils/transport/conn.go +++ b/xds/utils/transport/conn.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package transport import ( diff --git a/xds/utils/transport/networktype/networktype.go b/xds/utils/transport/networktype/networktype.go index b3686557fe..bfcb475d8d 100644 --- a/xds/utils/transport/networktype/networktype.go +++ b/xds/utils/transport/networktype/networktype.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package networktype declares the network type to be used in the default // dialer. Attribute of a resolver.Address. package networktype diff --git a/xds/utils/wrr/edf.go b/xds/utils/wrr/edf.go index 382192f923..6b9450364b 100644 --- a/xds/utils/wrr/edf.go +++ b/xds/utils/wrr/edf.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package wrr import ( diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go index cd591464f9..03cad735e1 100644 --- a/xds/utils/wrr/random.go +++ b/xds/utils/wrr/random.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package wrr import ( diff --git a/xds/utils/wrr/wrr.go b/xds/utils/wrr/wrr.go index 86402960e1..8e697deeb6 100644 --- a/xds/utils/wrr/wrr.go +++ b/xds/utils/wrr/wrr.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + // Package wrr contains the interface and common implementations of wrr // algorithms. package wrr diff --git a/xds/utils/xds_cache/timeoutCache.go b/xds/utils/xds_cache/timeoutCache.go index 56098da23f..69e342ab9e 100644 --- a/xds/utils/xds_cache/timeoutCache.go +++ b/xds/utils/xds_cache/timeoutCache.go @@ -14,6 +14,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +/* + * Copyright gRPC authors. + */ package xds_cache import ( diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index cc38b63769..ec850af186 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -15,6 +15,10 @@ * limitations under the License. */ +/* + * Copyright gRPC authors. + */ + package xds import ( From eb7230ab22f41de530265b645f4c5d2d879c7253 Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Fri, 1 Apr 2022 14:01:39 +0800 Subject: [PATCH 14/19] fix: grpc copyright --- xds/balancer/balancer.go | 4 +++- xds/balancer/cdsbalancer/cdsbalancer.go | 4 +++- xds/balancer/cdsbalancer/cluster_handler.go | 4 +++- xds/balancer/cdsbalancer/logging.go | 4 +++- xds/balancer/clusterimpl/clusterimpl.go | 4 +++- xds/balancer/clusterimpl/config.go | 4 +++- xds/balancer/clusterimpl/config_test.go | 4 +++- xds/balancer/clusterimpl/logging.go | 4 +++- xds/balancer/clusterimpl/picker.go | 4 +++- xds/balancer/clustermanager/balancerstateaggregator.go | 4 +++- xds/balancer/clustermanager/clustermanager.go | 4 +++- xds/balancer/clustermanager/config.go | 4 +++- xds/balancer/clustermanager/picker.go | 4 +++- xds/balancer/clusterresolver/clusterresolver.go | 4 +++- xds/balancer/clusterresolver/config.go | 4 +++- xds/balancer/clusterresolver/configbuilder.go | 4 +++- xds/balancer/clusterresolver/logging.go | 4 +++- xds/balancer/clusterresolver/resource_resolver.go | 4 +++- xds/balancer/clusterresolver/resource_resolver_dns.go | 4 +++- xds/balancer/clusterresolver/weightedtarget_config.go | 4 +++- xds/balancer/loadstore/load_store_wrapper.go | 4 +++- xds/balancer/orca/orca.go | 5 ++++- xds/balancer/priority/balancer.go | 4 +++- xds/balancer/priority/balancer_child.go | 4 +++- xds/balancer/priority/balancer_priority.go | 4 +++- xds/balancer/priority/config.go | 4 +++- xds/balancer/priority/config_test.go | 4 +++- xds/balancer/priority/ignore_resolve_now.go | 4 +++- xds/balancer/priority/logging.go | 4 +++- xds/balancer/priority/utils.go | 4 +++- xds/balancer/priority/utils_test.go | 4 +++- xds/balancer/ringhash/config.go | 4 +++- xds/balancer/ringhash/config_test.go | 4 +++- xds/balancer/ringhash/logging.go | 4 +++- xds/balancer/ringhash/picker.go | 4 +++- xds/balancer/ringhash/ring.go | 4 +++- xds/balancer/ringhash/ring_test.go | 4 +++- xds/balancer/ringhash/ringhash.go | 4 +++- xds/balancer/ringhash/util.go | 4 +++- xds/client/attributes.go | 4 +++- xds/client/authority.go | 4 +++- xds/client/bootstrap/bootstrap.go | 4 +++- xds/client/bootstrap/bootstrap_test.go | 4 +++- xds/client/bootstrap/logging.go | 4 +++- xds/client/bootstrap/template.go | 4 +++- xds/client/bootstrap/template_test.go | 4 +++- xds/client/client.go | 4 +++- xds/client/controller.go | 4 +++- xds/client/controller/controller.go | 4 +++- xds/client/controller/loadreport.go | 4 +++- xds/client/controller/transport.go | 4 +++- xds/client/controller/version/v2/client.go | 4 +++- xds/client/controller/version/v2/loadreport.go | 4 +++- xds/client/controller/version/v3/client.go | 4 +++- xds/client/controller/version/v3/loadreport.go | 4 +++- xds/client/controller/version/version.go | 4 +++- xds/client/dump.go | 4 +++- xds/client/load/reporter.go | 4 +++- xds/client/load/store.go | 4 +++- xds/client/load/store_test.go | 4 +++- xds/client/loadreport.go | 4 +++- xds/client/logging.go | 4 +++- xds/client/pubsub/dump.go | 4 +++- xds/client/pubsub/interface.go | 4 +++- xds/client/pubsub/pubsub.go | 4 +++- xds/client/pubsub/update.go | 4 +++- xds/client/pubsub/watch.go | 4 +++- xds/client/requests_counter.go | 4 +++- xds/client/resource/errors.go | 4 +++- xds/client/resource/filter_chain.go | 4 +++- xds/client/resource/locality_id.go | 4 +++- xds/client/resource/matcher.go | 4 +++- xds/client/resource/matcher_path.go | 4 +++- xds/client/resource/name.go | 4 +++- xds/client/resource/type.go | 4 +++- xds/client/resource/type_cds.go | 4 +++- xds/client/resource/type_eds.go | 4 +++- xds/client/resource/type_lds.go | 4 +++- xds/client/resource/type_rds.go | 4 +++- xds/client/resource/unmarshal.go | 4 +++- xds/client/resource/unmarshal_cds.go | 4 +++- xds/client/resource/unmarshal_eds.go | 4 +++- xds/client/resource/unmarshal_lds.go | 4 +++- xds/client/resource/unmarshal_rds.go | 4 +++- xds/client/resource/version/version.go | 4 +++- xds/client/singleton.go | 4 +++- xds/client/watchers.go | 4 +++- xds/clusterspecifier/cluster_specifier.go | 4 +++- xds/csds/csds.go | 4 +++- xds/httpfilter/fault/fault.go | 4 +++- xds/httpfilter/httpfilter.go | 4 +++- xds/httpfilter/rbac/rbac.go | 4 +++- xds/httpfilter/router/router.go | 4 +++- xds/internal/internal.go | 4 +++- xds/resolver/logging.go | 4 +++- xds/resolver/serviceconfig.go | 4 +++- xds/resolver/watch_service.go | 4 +++- xds/resolver/xds_resolver.go | 4 +++- xds/server/conn_wrapper.go | 4 +++- xds/server/listener_wrapper.go | 4 +++- xds/server/rds_handler.go | 4 +++- xds/utils/backoff/backoff.go | 4 +++- xds/utils/balancer/stub/stub.go | 4 +++- xds/utils/balancergroup/balancergroup.go | 5 ++++- xds/utils/balancergroup/balancerstateaggregator.go | 4 +++- xds/utils/balancerload/load.go | 4 +++- xds/utils/buffer/unbounded.go | 4 +++- xds/utils/credentials/xds/handshake_info.go | 4 +++- xds/utils/credentials/xds/handshake_info_test.go | 4 +++- xds/utils/envconfig/envconfig.go | 4 +++- xds/utils/envconfig/xds.go | 4 +++- xds/utils/grpclog/grpclog.go | 4 +++- xds/utils/grpclog/prefixLogger.go | 4 +++- xds/utils/grpcrand/grpcrand.go | 4 +++- xds/utils/grpcsync/event.go | 4 +++- xds/utils/grpcutil/encode_duration.go | 4 +++- xds/utils/grpcutil/encode_duration_test.go | 4 +++- xds/utils/grpcutil/grpcutil.go | 4 +++- xds/utils/grpcutil/metadata.go | 4 +++- xds/utils/grpcutil/method.go | 4 +++- xds/utils/grpcutil/method_test.go | 4 +++- xds/utils/grpcutil/regex.go | 4 +++- xds/utils/grpcutil/regex_test.go | 4 +++- xds/utils/hierarchy/hierarchy.go | 4 +++- xds/utils/hierarchy/hierarchy_test.go | 4 +++- xds/utils/matcher/matcher_header.go | 4 +++- xds/utils/matcher/matcher_header_test.go | 4 +++- xds/utils/matcher/regex.go | 4 +++- xds/utils/matcher/regex_test.go | 4 +++- xds/utils/matcher/string_matcher.go | 4 +++- xds/utils/matcher/string_matcher_test.go | 4 +++- xds/utils/metadata/metadata.go | 4 +++- xds/utils/pretty/pretty.go | 4 +++- xds/utils/rbac/matchers.go | 4 +++- xds/utils/rbac/rbac_engine.go | 4 +++- xds/utils/resolver/config_selector.go | 4 +++- xds/utils/resolver/passthrough/passthrough.go | 4 +++- xds/utils/resolver/unix/unix.go | 4 +++- xds/utils/serviceconfig/serviceconfig.go | 4 +++- xds/utils/serviceconfig/serviceconfig_test.go | 4 +++- xds/utils/transport/conn.go | 4 +++- xds/utils/transport/networktype/networktype.go | 4 +++- xds/utils/wrr/edf.go | 4 +++- xds/utils/wrr/random.go | 4 +++- xds/utils/wrr/wrr.go | 4 +++- xds/utils/xds_cache/timeoutCache.go | 5 ++++- xds/xds_handshake_cluster.go | 4 +++- 147 files changed, 444 insertions(+), 147 deletions(-) diff --git a/xds/balancer/balancer.go b/xds/balancer/balancer.go index 367e51f720..de005ad2f7 100644 --- a/xds/balancer/balancer.go +++ b/xds/balancer/balancer.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package balancer installs all the xds balancers. diff --git a/xds/balancer/cdsbalancer/cdsbalancer.go b/xds/balancer/cdsbalancer/cdsbalancer.go index d2e71d2b76..753a7d63ea 100644 --- a/xds/balancer/cdsbalancer/cdsbalancer.go +++ b/xds/balancer/cdsbalancer/cdsbalancer.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package cdsbalancer implements a balancer to handle CDS responses. diff --git a/xds/balancer/cdsbalancer/cluster_handler.go b/xds/balancer/cdsbalancer/cluster_handler.go index 76e657fe9d..20b5de1b77 100644 --- a/xds/balancer/cdsbalancer/cluster_handler.go +++ b/xds/balancer/cdsbalancer/cluster_handler.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package cdsbalancer diff --git a/xds/balancer/cdsbalancer/logging.go b/xds/balancer/cdsbalancer/logging.go index 6dd48b9f65..c1a673a7a4 100644 --- a/xds/balancer/cdsbalancer/logging.go +++ b/xds/balancer/cdsbalancer/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package cdsbalancer diff --git a/xds/balancer/clusterimpl/clusterimpl.go b/xds/balancer/clusterimpl/clusterimpl.go index 536c6c756e..61ae145b13 100644 --- a/xds/balancer/clusterimpl/clusterimpl.go +++ b/xds/balancer/clusterimpl/clusterimpl.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package clusterimpl implements the xds_cluster_impl balancing policy. It diff --git a/xds/balancer/clusterimpl/config.go b/xds/balancer/clusterimpl/config.go index ad50044108..d113b737fb 100644 --- a/xds/balancer/clusterimpl/config.go +++ b/xds/balancer/clusterimpl/config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/config_test.go b/xds/balancer/clusterimpl/config_test.go index c8c4b6e279..8e19364235 100644 --- a/xds/balancer/clusterimpl/config_test.go +++ b/xds/balancer/clusterimpl/config_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/logging.go b/xds/balancer/clusterimpl/logging.go index 3dd733ac63..3fba051760 100644 --- a/xds/balancer/clusterimpl/logging.go +++ b/xds/balancer/clusterimpl/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterimpl diff --git a/xds/balancer/clusterimpl/picker.go b/xds/balancer/clusterimpl/picker.go index 001567cb5e..afcd8b3a40 100644 --- a/xds/balancer/clusterimpl/picker.go +++ b/xds/balancer/clusterimpl/picker.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterimpl diff --git a/xds/balancer/clustermanager/balancerstateaggregator.go b/xds/balancer/clustermanager/balancerstateaggregator.go index 9712eaa0e4..8261d90a60 100644 --- a/xds/balancer/clustermanager/balancerstateaggregator.go +++ b/xds/balancer/clustermanager/balancerstateaggregator.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clustermanager diff --git a/xds/balancer/clustermanager/clustermanager.go b/xds/balancer/clustermanager/clustermanager.go index d14d2b2a68..41c63151ff 100644 --- a/xds/balancer/clustermanager/clustermanager.go +++ b/xds/balancer/clustermanager/clustermanager.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package clustermanager implements the cluster manager LB policy for xds. diff --git a/xds/balancer/clustermanager/config.go b/xds/balancer/clustermanager/config.go index 15cd28cbfa..738dc8f269 100644 --- a/xds/balancer/clustermanager/config.go +++ b/xds/balancer/clustermanager/config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clustermanager diff --git a/xds/balancer/clustermanager/picker.go b/xds/balancer/clustermanager/picker.go index 5d6b41b4a4..787fc1fa11 100644 --- a/xds/balancer/clustermanager/picker.go +++ b/xds/balancer/clustermanager/picker.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clustermanager diff --git a/xds/balancer/clusterresolver/clusterresolver.go b/xds/balancer/clusterresolver/clusterresolver.go index 6e3885ab07..b531368038 100644 --- a/xds/balancer/clusterresolver/clusterresolver.go +++ b/xds/balancer/clusterresolver/clusterresolver.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package clusterresolver contains EDS balancer implementation. diff --git a/xds/balancer/clusterresolver/config.go b/xds/balancer/clusterresolver/config.go index feb322af88..5a6d573073 100644 --- a/xds/balancer/clusterresolver/config.go +++ b/xds/balancer/clusterresolver/config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/configbuilder.go b/xds/balancer/clusterresolver/configbuilder.go index 8394e6adc4..a9397f487f 100644 --- a/xds/balancer/clusterresolver/configbuilder.go +++ b/xds/balancer/clusterresolver/configbuilder.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/logging.go b/xds/balancer/clusterresolver/logging.go index 35540826c7..c9822dae71 100644 --- a/xds/balancer/clusterresolver/logging.go +++ b/xds/balancer/clusterresolver/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/resource_resolver.go b/xds/balancer/clusterresolver/resource_resolver.go index 0b0fa3c26f..a5582f23f5 100644 --- a/xds/balancer/clusterresolver/resource_resolver.go +++ b/xds/balancer/clusterresolver/resource_resolver.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/resource_resolver_dns.go b/xds/balancer/clusterresolver/resource_resolver_dns.go index 33e21894c7..baabab38f9 100644 --- a/xds/balancer/clusterresolver/resource_resolver_dns.go +++ b/xds/balancer/clusterresolver/resource_resolver_dns.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/clusterresolver/weightedtarget_config.go b/xds/balancer/clusterresolver/weightedtarget_config.go index b3c4cad7a1..36811f63bb 100644 --- a/xds/balancer/clusterresolver/weightedtarget_config.go +++ b/xds/balancer/clusterresolver/weightedtarget_config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package clusterresolver diff --git a/xds/balancer/loadstore/load_store_wrapper.go b/xds/balancer/loadstore/load_store_wrapper.go index 992118fcf5..d3820a147a 100644 --- a/xds/balancer/loadstore/load_store_wrapper.go +++ b/xds/balancer/loadstore/load_store_wrapper.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package loadstore contains the loadStoreWrapper shared by the balancers. diff --git a/xds/balancer/orca/orca.go b/xds/balancer/orca/orca.go index 930e8e7588..4e78f1424e 100644 --- a/xds/balancer/orca/orca.go +++ b/xds/balancer/orca/orca.go @@ -16,8 +16,11 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ + // Package orca implements Open Request Cost Aggregation. package orca diff --git a/xds/balancer/priority/balancer.go b/xds/balancer/priority/balancer.go index 9386bc027c..99eb571620 100644 --- a/xds/balancer/priority/balancer.go +++ b/xds/balancer/priority/balancer.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package priority implements the priority balancer. diff --git a/xds/balancer/priority/balancer_child.go b/xds/balancer/priority/balancer_child.go index 7517999eeb..d7414c25ad 100644 --- a/xds/balancer/priority/balancer_child.go +++ b/xds/balancer/priority/balancer_child.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/balancer_priority.go b/xds/balancer/priority/balancer_priority.go index aa966fc304..eafdbf2da7 100644 --- a/xds/balancer/priority/balancer_priority.go +++ b/xds/balancer/priority/balancer_priority.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/config.go b/xds/balancer/priority/config.go index 2f79221f1f..aa5bf37ca9 100644 --- a/xds/balancer/priority/config.go +++ b/xds/balancer/priority/config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/config_test.go b/xds/balancer/priority/config_test.go index 165f7a5cd5..41e8f79067 100644 --- a/xds/balancer/priority/config_test.go +++ b/xds/balancer/priority/config_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/ignore_resolve_now.go b/xds/balancer/priority/ignore_resolve_now.go index 07838660b6..5102d2e057 100644 --- a/xds/balancer/priority/ignore_resolve_now.go +++ b/xds/balancer/priority/ignore_resolve_now.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/logging.go b/xds/balancer/priority/logging.go index a4d870cecf..438f03da82 100644 --- a/xds/balancer/priority/logging.go +++ b/xds/balancer/priority/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/utils.go b/xds/balancer/priority/utils.go index ea2956d6f8..5404a8d259 100644 --- a/xds/balancer/priority/utils.go +++ b/xds/balancer/priority/utils.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/priority/utils_test.go b/xds/balancer/priority/utils_test.go index 027222b869..3669d5ecb9 100644 --- a/xds/balancer/priority/utils_test.go +++ b/xds/balancer/priority/utils_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package priority diff --git a/xds/balancer/ringhash/config.go b/xds/balancer/ringhash/config.go index ec94fe89cb..0154fe5a47 100644 --- a/xds/balancer/ringhash/config.go +++ b/xds/balancer/ringhash/config.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/config_test.go b/xds/balancer/ringhash/config_test.go index c9f8fc7fed..256d20beca 100644 --- a/xds/balancer/ringhash/config_test.go +++ b/xds/balancer/ringhash/config_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/logging.go b/xds/balancer/ringhash/logging.go index 2265075f79..56e896ced7 100644 --- a/xds/balancer/ringhash/logging.go +++ b/xds/balancer/ringhash/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/picker.go b/xds/balancer/ringhash/picker.go index 59f93123fc..b004e3a4bb 100644 --- a/xds/balancer/ringhash/picker.go +++ b/xds/balancer/ringhash/picker.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/ring.go b/xds/balancer/ringhash/ring.go index cb5d861ba0..fc546fa6c3 100644 --- a/xds/balancer/ringhash/ring.go +++ b/xds/balancer/ringhash/ring.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/ring_test.go b/xds/balancer/ringhash/ring_test.go index e9d967a4cd..997a4adea0 100644 --- a/xds/balancer/ringhash/ring_test.go +++ b/xds/balancer/ringhash/ring_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/balancer/ringhash/ringhash.go b/xds/balancer/ringhash/ringhash.go index cfc7101bdc..76ff0725da 100644 --- a/xds/balancer/ringhash/ringhash.go +++ b/xds/balancer/ringhash/ringhash.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package ringhash implements the ringhash balancer. diff --git a/xds/balancer/ringhash/util.go b/xds/balancer/ringhash/util.go index 2db9d21c93..a3b0b54e5f 100644 --- a/xds/balancer/ringhash/util.go +++ b/xds/balancer/ringhash/util.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package ringhash diff --git a/xds/client/attributes.go b/xds/client/attributes.go index 97d536a196..294ee6c894 100644 --- a/xds/client/attributes.go +++ b/xds/client/attributes.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package client diff --git a/xds/client/authority.go b/xds/client/authority.go index 50abb515e1..518d04dfca 100644 --- a/xds/client/authority.go +++ b/xds/client/authority.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package client diff --git a/xds/client/bootstrap/bootstrap.go b/xds/client/bootstrap/bootstrap.go index 2ce5410a13..de80feb992 100644 --- a/xds/client/bootstrap/bootstrap.go +++ b/xds/client/bootstrap/bootstrap.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package bootstrap provides the functionality to initialize certain aspects diff --git a/xds/client/bootstrap/bootstrap_test.go b/xds/client/bootstrap/bootstrap_test.go index 2d9ce1220e..64dd0bc50b 100644 --- a/xds/client/bootstrap/bootstrap_test.go +++ b/xds/client/bootstrap/bootstrap_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package bootstrap diff --git a/xds/client/bootstrap/logging.go b/xds/client/bootstrap/logging.go index adeb240b3f..0dc023e25c 100644 --- a/xds/client/bootstrap/logging.go +++ b/xds/client/bootstrap/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package bootstrap diff --git a/xds/client/bootstrap/template.go b/xds/client/bootstrap/template.go index e88fec4410..936dd79a82 100644 --- a/xds/client/bootstrap/template.go +++ b/xds/client/bootstrap/template.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package bootstrap diff --git a/xds/client/bootstrap/template_test.go b/xds/client/bootstrap/template_test.go index cdb125a1d5..a02504b65c 100644 --- a/xds/client/bootstrap/template_test.go +++ b/xds/client/bootstrap/template_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package bootstrap diff --git a/xds/client/client.go b/xds/client/client.go index 9aff7e7df9..fbf9bfcfd9 100644 --- a/xds/client/client.go +++ b/xds/client/client.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // package client implements a full fledged gRPC client for the xDS API used diff --git a/xds/client/controller.go b/xds/client/controller.go index 509079664f..29d2cd726f 100644 --- a/xds/client/controller.go +++ b/xds/client/controller.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package client diff --git a/xds/client/controller/controller.go b/xds/client/controller/controller.go index 6fc6b75b56..eb167d7ebc 100644 --- a/xds/client/controller/controller.go +++ b/xds/client/controller/controller.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package controller contains implementation to connect to the control plane. diff --git a/xds/client/controller/loadreport.go b/xds/client/controller/loadreport.go index 25c0845411..14916283c5 100644 --- a/xds/client/controller/loadreport.go +++ b/xds/client/controller/loadreport.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package controller diff --git a/xds/client/controller/transport.go b/xds/client/controller/transport.go index 5a5ce315c7..926f0dda9e 100644 --- a/xds/client/controller/transport.go +++ b/xds/client/controller/transport.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package controller diff --git a/xds/client/controller/version/v2/client.go b/xds/client/controller/version/v2/client.go index b970c85c27..ca44ed2f89 100644 --- a/xds/client/controller/version/v2/client.go +++ b/xds/client/controller/version/v2/client.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package v2 provides xDS v2 transport protocol specific functionality. diff --git a/xds/client/controller/version/v2/loadreport.go b/xds/client/controller/version/v2/loadreport.go index 37f44bd429..35947e9ad2 100644 --- a/xds/client/controller/version/v2/loadreport.go +++ b/xds/client/controller/version/v2/loadreport.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package v2 diff --git a/xds/client/controller/version/v3/client.go b/xds/client/controller/version/v3/client.go index ce68cc1216..4f3e2f49de 100644 --- a/xds/client/controller/version/v3/client.go +++ b/xds/client/controller/version/v3/client.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package v3 provides xDS v3 transport protocol specific functionality. diff --git a/xds/client/controller/version/v3/loadreport.go b/xds/client/controller/version/v3/loadreport.go index 63824539ad..cedf8f6c08 100644 --- a/xds/client/controller/version/v3/loadreport.go +++ b/xds/client/controller/version/v3/loadreport.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package v3 diff --git a/xds/client/controller/version/version.go b/xds/client/controller/version/version.go index 62c7fb9c5c..38a91a9e7f 100644 --- a/xds/client/controller/version/version.go +++ b/xds/client/controller/version/version.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package version defines APIs to deal with different versions of xDS. diff --git a/xds/client/dump.go b/xds/client/dump.go index 1425dd371e..560190ab11 100644 --- a/xds/client/dump.go +++ b/xds/client/dump.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package client diff --git a/xds/client/load/reporter.go b/xds/client/load/reporter.go index 446206a5fb..4f489ee8f0 100644 --- a/xds/client/load/reporter.go +++ b/xds/client/load/reporter.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package load diff --git a/xds/client/load/store.go b/xds/client/load/store.go index 9aea03a5cf..6ae5c8500a 100644 --- a/xds/client/load/store.go +++ b/xds/client/load/store.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package load provides functionality to record and maintain load data. diff --git a/xds/client/load/store_test.go b/xds/client/load/store_test.go index 9148e82da6..94fe3c62ad 100644 --- a/xds/client/load/store_test.go +++ b/xds/client/load/store_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package load diff --git a/xds/client/loadreport.go b/xds/client/loadreport.go index ff02fb9a66..ad3f14aeb0 100644 --- a/xds/client/loadreport.go +++ b/xds/client/loadreport.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package client diff --git a/xds/client/logging.go b/xds/client/logging.go index 808d83712b..0f94d55829 100644 --- a/xds/client/logging.go +++ b/xds/client/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package client diff --git a/xds/client/pubsub/dump.go b/xds/client/pubsub/dump.go index d7f80667d3..27fe97e206 100644 --- a/xds/client/pubsub/dump.go +++ b/xds/client/pubsub/dump.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package pubsub diff --git a/xds/client/pubsub/interface.go b/xds/client/pubsub/interface.go index cf29dd018d..2ecd90e156 100644 --- a/xds/client/pubsub/interface.go +++ b/xds/client/pubsub/interface.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package pubsub diff --git a/xds/client/pubsub/pubsub.go b/xds/client/pubsub/pubsub.go index 678b9ee981..7318321bb7 100644 --- a/xds/client/pubsub/pubsub.go +++ b/xds/client/pubsub/pubsub.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package pubsub implements a utility type to maintain resource watchers and diff --git a/xds/client/pubsub/update.go b/xds/client/pubsub/update.go index 6b546ccc49..997eebc270 100644 --- a/xds/client/pubsub/update.go +++ b/xds/client/pubsub/update.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package pubsub diff --git a/xds/client/pubsub/watch.go b/xds/client/pubsub/watch.go index 5aec6757e7..75f5d1489f 100644 --- a/xds/client/pubsub/watch.go +++ b/xds/client/pubsub/watch.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package pubsub diff --git a/xds/client/requests_counter.go b/xds/client/requests_counter.go index 34d0cb4bd8..4db1919b5f 100644 --- a/xds/client/requests_counter.go +++ b/xds/client/requests_counter.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package client diff --git a/xds/client/resource/errors.go b/xds/client/resource/errors.go index 5b6ce7bbea..edac12eac1 100644 --- a/xds/client/resource/errors.go +++ b/xds/client/resource/errors.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/filter_chain.go b/xds/client/resource/filter_chain.go index 6a77e8db87..6ee8659447 100644 --- a/xds/client/resource/filter_chain.go +++ b/xds/client/resource/filter_chain.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/locality_id.go b/xds/client/resource/locality_id.go index d26dcedd58..2e222b1184 100644 --- a/xds/client/resource/locality_id.go +++ b/xds/client/resource/locality_id.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package internal contains functions/structs shared by xds diff --git a/xds/client/resource/matcher.go b/xds/client/resource/matcher.go index 6d203895d0..ab1cc6804c 100644 --- a/xds/client/resource/matcher.go +++ b/xds/client/resource/matcher.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/matcher_path.go b/xds/client/resource/matcher_path.go index 859b142cdf..ba7aab93b5 100644 --- a/xds/client/resource/matcher_path.go +++ b/xds/client/resource/matcher_path.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/name.go b/xds/client/resource/name.go index f8959ac7cc..d7693bd1bc 100644 --- a/xds/client/resource/name.go +++ b/xds/client/resource/name.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/type.go b/xds/client/resource/type.go index c71b0c233b..c473f1c611 100644 --- a/xds/client/resource/type.go +++ b/xds/client/resource/type.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/type_cds.go b/xds/client/resource/type_cds.go index 1c6b42ba20..a5e18e47ad 100644 --- a/xds/client/resource/type_cds.go +++ b/xds/client/resource/type_cds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/type_eds.go b/xds/client/resource/type_eds.go index 499e9491b0..b7c4b1ca19 100644 --- a/xds/client/resource/type_eds.go +++ b/xds/client/resource/type_eds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/type_lds.go b/xds/client/resource/type_lds.go index 490c804924..953252d98f 100644 --- a/xds/client/resource/type_lds.go +++ b/xds/client/resource/type_lds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/type_rds.go b/xds/client/resource/type_rds.go index 5312cfb49a..2edc93017a 100644 --- a/xds/client/resource/type_rds.go +++ b/xds/client/resource/type_rds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/unmarshal.go b/xds/client/resource/unmarshal.go index 7252df0d4c..23a2d50e51 100644 --- a/xds/client/resource/unmarshal.go +++ b/xds/client/resource/unmarshal.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package resource contains functions to proto xds updates (unmarshal from diff --git a/xds/client/resource/unmarshal_cds.go b/xds/client/resource/unmarshal_cds.go index 015104eb0a..2023d3170d 100644 --- a/xds/client/resource/unmarshal_cds.go +++ b/xds/client/resource/unmarshal_cds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/unmarshal_eds.go b/xds/client/resource/unmarshal_eds.go index 1f3921f2e8..93068fe04f 100644 --- a/xds/client/resource/unmarshal_eds.go +++ b/xds/client/resource/unmarshal_eds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/unmarshal_lds.go b/xds/client/resource/unmarshal_lds.go index 624a72ef40..258dc3ac59 100644 --- a/xds/client/resource/unmarshal_lds.go +++ b/xds/client/resource/unmarshal_lds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/unmarshal_rds.go b/xds/client/resource/unmarshal_rds.go index f155cbee8f..7813cbdc1d 100644 --- a/xds/client/resource/unmarshal_rds.go +++ b/xds/client/resource/unmarshal_rds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package resource diff --git a/xds/client/resource/version/version.go b/xds/client/resource/version/version.go index 93c7127e5e..b862619c48 100644 --- a/xds/client/resource/version/version.go +++ b/xds/client/resource/version/version.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package version defines constants to distinguish between supported xDS API diff --git a/xds/client/singleton.go b/xds/client/singleton.go index 8a8c17b654..0f760d17a0 100644 --- a/xds/client/singleton.go +++ b/xds/client/singleton.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package client diff --git a/xds/client/watchers.go b/xds/client/watchers.go index cd9c3d2954..614da93924 100644 --- a/xds/client/watchers.go +++ b/xds/client/watchers.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package client diff --git a/xds/clusterspecifier/cluster_specifier.go b/xds/clusterspecifier/cluster_specifier.go index 6afca4083d..81c0d076fb 100644 --- a/xds/clusterspecifier/cluster_specifier.go +++ b/xds/clusterspecifier/cluster_specifier.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package clusterspecifier contains the ClusterSpecifier interface and a registry for diff --git a/xds/csds/csds.go b/xds/csds/csds.go index 818e13c01b..8d5fce22e4 100644 --- a/xds/csds/csds.go +++ b/xds/csds/csds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package csds implements features to dump the status (xDS responses) the diff --git a/xds/httpfilter/fault/fault.go b/xds/httpfilter/fault/fault.go index 04b45e5096..7b92cf2e5a 100644 --- a/xds/httpfilter/fault/fault.go +++ b/xds/httpfilter/fault/fault.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package fault implements the Envoy Fault Injection HTTP filter. diff --git a/xds/httpfilter/httpfilter.go b/xds/httpfilter/httpfilter.go index a82794b6aa..d849a4f09f 100644 --- a/xds/httpfilter/httpfilter.go +++ b/xds/httpfilter/httpfilter.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package httpfilter contains the HTTPFilter interface and a registry for diff --git a/xds/httpfilter/rbac/rbac.go b/xds/httpfilter/rbac/rbac.go index 37b2aa7e21..90085b0119 100644 --- a/xds/httpfilter/rbac/rbac.go +++ b/xds/httpfilter/rbac/rbac.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package rbac implements the Envoy RBAC HTTP filter. diff --git a/xds/httpfilter/router/router.go b/xds/httpfilter/router/router.go index 41d4f05e8b..cb40131a80 100644 --- a/xds/httpfilter/router/router.go +++ b/xds/httpfilter/router/router.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package router implements the Envoy Router HTTP filter. diff --git a/xds/internal/internal.go b/xds/internal/internal.go index 0da8c9d068..544ff3cad9 100644 --- a/xds/internal/internal.go +++ b/xds/internal/internal.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2016 gRPC authors. + * */ // Package internal contains gRPC-internal code, to avoid polluting diff --git a/xds/resolver/logging.go b/xds/resolver/logging.go index f42a482849..35006edb49 100644 --- a/xds/resolver/logging.go +++ b/xds/resolver/logging.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package resolver diff --git a/xds/resolver/serviceconfig.go b/xds/resolver/serviceconfig.go index 3f0a6c797b..f941553804 100644 --- a/xds/resolver/serviceconfig.go +++ b/xds/resolver/serviceconfig.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package resolver diff --git a/xds/resolver/watch_service.go b/xds/resolver/watch_service.go index e407b3a0e6..3ef4f844be 100644 --- a/xds/resolver/watch_service.go +++ b/xds/resolver/watch_service.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package resolver diff --git a/xds/resolver/xds_resolver.go b/xds/resolver/xds_resolver.go index fa22475e10..2d597d1d41 100644 --- a/xds/resolver/xds_resolver.go +++ b/xds/resolver/xds_resolver.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package resolver implements the xds resolver, that does LDS and RDS to find diff --git a/xds/server/conn_wrapper.go b/xds/server/conn_wrapper.go index ed9b0dd1ea..421a1bdd2f 100644 --- a/xds/server/conn_wrapper.go +++ b/xds/server/conn_wrapper.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package server diff --git a/xds/server/listener_wrapper.go b/xds/server/listener_wrapper.go index db4a214ac6..27ffc64bc2 100644 --- a/xds/server/listener_wrapper.go +++ b/xds/server/listener_wrapper.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package server contains internal server-side functionality used by the public diff --git a/xds/server/rds_handler.go b/xds/server/rds_handler.go index f85cf0b200..62b3a9eb60 100644 --- a/xds/server/rds_handler.go +++ b/xds/server/rds_handler.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package server diff --git a/xds/utils/backoff/backoff.go b/xds/utils/backoff/backoff.go index 270b2c60c6..f93a9363f8 100644 --- a/xds/utils/backoff/backoff.go +++ b/xds/utils/backoff/backoff.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2017 gRPC authors. + * */ // Package backoff implement the backoff strategy for gRPC. diff --git a/xds/utils/balancer/stub/stub.go b/xds/utils/balancer/stub/stub.go index bfd52209c1..8e392340bf 100644 --- a/xds/utils/balancer/stub/stub.go +++ b/xds/utils/balancer/stub/stub.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package stub implements a balancer for testing purposes. diff --git a/xds/utils/balancergroup/balancergroup.go b/xds/utils/balancergroup/balancergroup.go index f9fb9069f6..2e0dc8660c 100644 --- a/xds/utils/balancergroup/balancergroup.go +++ b/xds/utils/balancergroup/balancergroup.go @@ -16,8 +16,11 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ + // Package balancergroup implements a utility struct to bind multiple balancers // into one balancer. package balancergroup diff --git a/xds/utils/balancergroup/balancerstateaggregator.go b/xds/utils/balancergroup/balancerstateaggregator.go index 6c3273fe70..c5b809d3da 100644 --- a/xds/utils/balancergroup/balancerstateaggregator.go +++ b/xds/utils/balancergroup/balancerstateaggregator.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package balancergroup diff --git a/xds/utils/balancerload/load.go b/xds/utils/balancerload/load.go index baf10c41b9..54e5f0a62c 100644 --- a/xds/utils/balancerload/load.go +++ b/xds/utils/balancerload/load.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package balancerload defines APIs to parse server loads in trailers. The // parsed loads are sent to balancers in DoneInfo. diff --git a/xds/utils/buffer/unbounded.go b/xds/utils/buffer/unbounded.go index 85b0de6262..f5bcf4782e 100644 --- a/xds/utils/buffer/unbounded.go +++ b/xds/utils/buffer/unbounded.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package buffer provides an implementation of an unbounded buffer. diff --git a/xds/utils/credentials/xds/handshake_info.go b/xds/utils/credentials/xds/handshake_info.go index 12437db40d..a7d62ab5fd 100644 --- a/xds/utils/credentials/xds/handshake_info.go +++ b/xds/utils/credentials/xds/handshake_info.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package xds contains non-user facing functionality of the xds credentials. diff --git a/xds/utils/credentials/xds/handshake_info_test.go b/xds/utils/credentials/xds/handshake_info_test.go index 58893d44f5..69f250979d 100644 --- a/xds/utils/credentials/xds/handshake_info_test.go +++ b/xds/utils/credentials/xds/handshake_info_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package xds diff --git a/xds/utils/envconfig/envconfig.go b/xds/utils/envconfig/envconfig.go index 119062fa68..8ba5979cdb 100644 --- a/xds/utils/envconfig/envconfig.go +++ b/xds/utils/envconfig/envconfig.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2018 gRPC authors. + * */ // Package envconfig contains grpc settings configured by environment variables. diff --git a/xds/utils/envconfig/xds.go b/xds/utils/envconfig/xds.go index 7f6c560258..90ff78f083 100644 --- a/xds/utils/envconfig/xds.go +++ b/xds/utils/envconfig/xds.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package envconfig diff --git a/xds/utils/grpclog/grpclog.go b/xds/utils/grpclog/grpclog.go index 89bbdc775a..b585fbff14 100644 --- a/xds/utils/grpclog/grpclog.go +++ b/xds/utils/grpclog/grpclog.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package grpclog (internal) defines depth logging for grpc. diff --git a/xds/utils/grpclog/prefixLogger.go b/xds/utils/grpclog/prefixLogger.go index bbbcc786c3..46abf112fa 100644 --- a/xds/utils/grpclog/prefixLogger.go +++ b/xds/utils/grpclog/prefixLogger.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package grpclog diff --git a/xds/utils/grpcrand/grpcrand.go b/xds/utils/grpcrand/grpcrand.go index a026c890c2..0fc9a92b93 100644 --- a/xds/utils/grpcrand/grpcrand.go +++ b/xds/utils/grpcrand/grpcrand.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2018 gRPC authors. + * */ // Package grpcrand implements math/rand functions in a concurrent-safe way diff --git a/xds/utils/grpcsync/event.go b/xds/utils/grpcsync/event.go index 69426ad0b2..c374e2c9f3 100644 --- a/xds/utils/grpcsync/event.go +++ b/xds/utils/grpcsync/event.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2018 gRPC authors. + * */ // Package grpcsync implements additional synchronization primitives built upon diff --git a/xds/utils/grpcutil/encode_duration.go b/xds/utils/grpcutil/encode_duration.go index 613a9f2ef3..972d2fce34 100644 --- a/xds/utils/grpcutil/encode_duration.go +++ b/xds/utils/grpcutil/encode_duration.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/encode_duration_test.go b/xds/utils/grpcutil/encode_duration_test.go index c77c80139c..085e0bcd2d 100644 --- a/xds/utils/grpcutil/encode_duration_test.go +++ b/xds/utils/grpcutil/encode_duration_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/grpcutil.go b/xds/utils/grpcutil/grpcutil.go index 211ee64a6b..a2a78bb4be 100644 --- a/xds/utils/grpcutil/grpcutil.go +++ b/xds/utils/grpcutil/grpcutil.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package grpcutil provides utility functions used across the gRPC codebase. diff --git a/xds/utils/grpcutil/metadata.go b/xds/utils/grpcutil/metadata.go index 87fe9d3d21..20c5fc0c5a 100644 --- a/xds/utils/grpcutil/metadata.go +++ b/xds/utils/grpcutil/metadata.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/method.go b/xds/utils/grpcutil/method.go index 369819ce83..a112081445 100644 --- a/xds/utils/grpcutil/method.go +++ b/xds/utils/grpcutil/method.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2018 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/method_test.go b/xds/utils/grpcutil/method_test.go index cfd0adc3e6..76117f536b 100644 --- a/xds/utils/grpcutil/method_test.go +++ b/xds/utils/grpcutil/method_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2018 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/regex.go b/xds/utils/grpcutil/regex.go index d12ae5f5ac..94b997325a 100644 --- a/xds/utils/grpcutil/regex.go +++ b/xds/utils/grpcutil/regex.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/grpcutil/regex_test.go b/xds/utils/grpcutil/regex_test.go index 8137f7cb0a..8e6fc75374 100644 --- a/xds/utils/grpcutil/regex_test.go +++ b/xds/utils/grpcutil/regex_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package grpcutil diff --git a/xds/utils/hierarchy/hierarchy.go b/xds/utils/hierarchy/hierarchy.go index 6bb8e7065f..4833f7df37 100644 --- a/xds/utils/hierarchy/hierarchy.go +++ b/xds/utils/hierarchy/hierarchy.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package hierarchy contains functions to set and get hierarchy string from diff --git a/xds/utils/hierarchy/hierarchy_test.go b/xds/utils/hierarchy/hierarchy_test.go index 8629dd8082..b53728bddd 100644 --- a/xds/utils/hierarchy/hierarchy_test.go +++ b/xds/utils/hierarchy/hierarchy_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package hierarchy diff --git a/xds/utils/matcher/matcher_header.go b/xds/utils/matcher/matcher_header.go index cc3a94e56d..826a396b03 100644 --- a/xds/utils/matcher/matcher_header.go +++ b/xds/utils/matcher/matcher_header.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package matcher diff --git a/xds/utils/matcher/matcher_header_test.go b/xds/utils/matcher/matcher_header_test.go index 12f4465496..b56abc58d7 100644 --- a/xds/utils/matcher/matcher_header_test.go +++ b/xds/utils/matcher/matcher_header_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package matcher diff --git a/xds/utils/matcher/regex.go b/xds/utils/matcher/regex.go index 77ef637c27..eea4fb3d41 100644 --- a/xds/utils/matcher/regex.go +++ b/xds/utils/matcher/regex.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package matcher diff --git a/xds/utils/matcher/regex_test.go b/xds/utils/matcher/regex_test.go index e5e3f9c077..080b26c445 100644 --- a/xds/utils/matcher/regex_test.go +++ b/xds/utils/matcher/regex_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package matcher diff --git a/xds/utils/matcher/string_matcher.go b/xds/utils/matcher/string_matcher.go index 7785af9839..3647a6fdb0 100644 --- a/xds/utils/matcher/string_matcher.go +++ b/xds/utils/matcher/string_matcher.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package matcher contains types that need to be shared between code under diff --git a/xds/utils/matcher/string_matcher_test.go b/xds/utils/matcher/string_matcher_test.go index 96731adb8c..48795e8bac 100644 --- a/xds/utils/matcher/string_matcher_test.go +++ b/xds/utils/matcher/string_matcher_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package matcher diff --git a/xds/utils/metadata/metadata.go b/xds/utils/metadata/metadata.go index cbb1dd4f16..3453a65441 100644 --- a/xds/utils/metadata/metadata.go +++ b/xds/utils/metadata/metadata.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package metadata diff --git a/xds/utils/pretty/pretty.go b/xds/utils/pretty/pretty.go index 523cc444fb..0acc838569 100644 --- a/xds/utils/pretty/pretty.go +++ b/xds/utils/pretty/pretty.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package pretty defines helper functions to pretty-print structs for logging. diff --git a/xds/utils/rbac/matchers.go b/xds/utils/rbac/matchers.go index 0a330a1417..32394ca7e8 100644 --- a/xds/utils/rbac/matchers.go +++ b/xds/utils/rbac/matchers.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package rbac diff --git a/xds/utils/rbac/rbac_engine.go b/xds/utils/rbac/rbac_engine.go index e8dd11cc11..b35f996326 100644 --- a/xds/utils/rbac/rbac_engine.go +++ b/xds/utils/rbac/rbac_engine.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package rbac provides service-level and method-level access control for a diff --git a/xds/utils/resolver/config_selector.go b/xds/utils/resolver/config_selector.go index 8a47062f4c..81e9c2ace1 100644 --- a/xds/utils/resolver/config_selector.go +++ b/xds/utils/resolver/config_selector.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package resolver provides internal resolver-related functionality. diff --git a/xds/utils/resolver/passthrough/passthrough.go b/xds/utils/resolver/passthrough/passthrough.go index 3447ab5459..0d9b6ae5fa 100644 --- a/xds/utils/resolver/passthrough/passthrough.go +++ b/xds/utils/resolver/passthrough/passthrough.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package passthrough implements a pass-through resolver. It sends the target diff --git a/xds/utils/resolver/unix/unix.go b/xds/utils/resolver/unix/unix.go index dbf9f18e9e..aa8ca136bc 100644 --- a/xds/utils/resolver/unix/unix.go +++ b/xds/utils/resolver/unix/unix.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package unix implements a resolver for unix targets. diff --git a/xds/utils/serviceconfig/serviceconfig.go b/xds/utils/serviceconfig/serviceconfig.go index 82cc7795bc..79f5f0e530 100644 --- a/xds/utils/serviceconfig/serviceconfig.go +++ b/xds/utils/serviceconfig/serviceconfig.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ // Package serviceconfig contains utility functions to parse service config. diff --git a/xds/utils/serviceconfig/serviceconfig_test.go b/xds/utils/serviceconfig/serviceconfig_test.go index ef34b3de02..5376b59f67 100644 --- a/xds/utils/serviceconfig/serviceconfig_test.go +++ b/xds/utils/serviceconfig/serviceconfig_test.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package serviceconfig diff --git a/xds/utils/transport/conn.go b/xds/utils/transport/conn.go index a7884d7cd5..78c78dbaad 100644 --- a/xds/utils/transport/conn.go +++ b/xds/utils/transport/conn.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ package transport diff --git a/xds/utils/transport/networktype/networktype.go b/xds/utils/transport/networktype/networktype.go index bfcb475d8d..1b6bed888f 100644 --- a/xds/utils/transport/networktype/networktype.go +++ b/xds/utils/transport/networktype/networktype.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2020 gRPC authors. + * */ // Package networktype declares the network type to be used in the default diff --git a/xds/utils/wrr/edf.go b/xds/utils/wrr/edf.go index 6b9450364b..47d8942e6a 100644 --- a/xds/utils/wrr/edf.go +++ b/xds/utils/wrr/edf.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package wrr diff --git a/xds/utils/wrr/random.go b/xds/utils/wrr/random.go index 03cad735e1..9f857f524d 100644 --- a/xds/utils/wrr/random.go +++ b/xds/utils/wrr/random.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ package wrr diff --git a/xds/utils/wrr/wrr.go b/xds/utils/wrr/wrr.go index 8e697deeb6..ddf9d658de 100644 --- a/xds/utils/wrr/wrr.go +++ b/xds/utils/wrr/wrr.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ // Package wrr contains the interface and common implementations of wrr diff --git a/xds/utils/xds_cache/timeoutCache.go b/xds/utils/xds_cache/timeoutCache.go index 69e342ab9e..2c7666e6fa 100644 --- a/xds/utils/xds_cache/timeoutCache.go +++ b/xds/utils/xds_cache/timeoutCache.go @@ -16,8 +16,11 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2019 gRPC authors. + * */ + package xds_cache import ( diff --git a/xds/xds_handshake_cluster.go b/xds/xds_handshake_cluster.go index ec850af186..a354a44249 100644 --- a/xds/xds_handshake_cluster.go +++ b/xds/xds_handshake_cluster.go @@ -16,7 +16,9 @@ */ /* - * Copyright gRPC authors. + * + * Copyright 2021 gRPC authors. + * */ package xds From 860212836a11fac7051a7a312103f5d53df7327a Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Mon, 4 Apr 2022 14:43:06 +0800 Subject: [PATCH 15/19] fix: xds comments --- .github/workflows/github-actions.yml | 2 +- .github/workflows/golangci-lint.yml | 2 +- cluster/router/meshrouter/meshrouter.go | 8 +- go.mod | 115 +----------------- go.sum | 6 + protocol/invocation.go | 3 +- protocol/invocation/rpcinvocation.go | 2 +- registry/xds/registry.go | 2 +- remoting/xds/client.go | 6 +- remoting/xds/common/model.go | 9 +- .../{interfaceMapping => mapping}/debug.go | 2 +- .../debug_test.go | 2 +- .../{interfaceMapping => mapping}/handler.go | 3 +- .../handler_test.go | 2 +- .../{interfaceMapping => mapping}/metadata.go | 2 +- .../mocks/InterfaceMapHandler.go | 0 remoting/xds/xds_client_factory.go | 4 +- 17 files changed, 33 insertions(+), 137 deletions(-) rename remoting/xds/{interfaceMapping => mapping}/debug.go (98%) rename remoting/xds/{interfaceMapping => mapping}/debug_test.go (99%) rename remoting/xds/{interfaceMapping => mapping}/handler.go (98%) rename remoting/xds/{interfaceMapping => mapping}/handler_test.go (99%) rename remoting/xds/{interfaceMapping => mapping}/metadata.go (98%) rename remoting/xds/{interfaceMapping => mapping}/mocks/InterfaceMapHandler.go (100%) diff --git a/.github/workflows/github-actions.yml b/.github/workflows/github-actions.yml index b3ba3dcfd5..f1c4676100 100644 --- a/.github/workflows/github-actions.yml +++ b/.github/workflows/github-actions.yml @@ -14,7 +14,7 @@ jobs: # If you want to matrix build , you can append the following list. matrix: go_version: - - 1.17 + - 1.15 os: - ubuntu-latest diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index 4c47124a12..ed4dcbc2ef 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: golang: - - 1.17 + - 1.15 steps: - uses: actions/setup-go@v2 with: diff --git a/cluster/router/meshrouter/meshrouter.go b/cluster/router/meshrouter/meshrouter.go index 44c3c08ce0..a39376d1b3 100644 --- a/cluster/router/meshrouter/meshrouter.go +++ b/cluster/router/meshrouter/meshrouter.go @@ -85,10 +85,11 @@ func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocat // 2. match http route for _, r := range vh.Routes { //route. - ctx := invocation.ToContext() + ctx := invocation.GetAttachmentAsContext() matcher, err := resource.RouteToMatcher(r) if err != nil { - panic(err) + logger.Errorf("[Mesh Router] router to matcher failed with error %s", err) + return invokers } if matcher.Match(resolver.RPCInfo{ Context: ctx, @@ -96,7 +97,8 @@ func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocat }) { // Loop through routes in order and select first match. if r == nil || r.WeightedClusters == nil { - panic("cluster notfound") + logger.Errorf("[Mesh Router] route's WeightedClusters is empty, route: %+v", r) + return invokers } invokersWeightPairs := make(invokerWeightPairs, 0) diff --git a/go.mod b/go.mod index deb0727230..e6c809698c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module dubbo.apache.org/dubbo-go/v3 -go 1.17 +go 1.15 require ( contrib.go.opencensus.io/exporter/prometheus v0.4.1 @@ -56,116 +56,3 @@ require ( k8s.io/apimachinery v0.22.4 k8s.io/client-go v0.16.9 ) - -require ( - cloud.google.com/go v0.65.0 // indirect - github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect - github.com/aliyun/alibaba-cloud-sdk-go v1.61.18 // indirect - github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.2.0 // indirect - github.com/buger/jsonparser v1.1.1 // indirect - github.com/census-instrumentation/opencensus-proto v0.2.1 // indirect - github.com/coreos/go-semver v0.3.0 // indirect - github.com/coreos/go-systemd/v22 v22.3.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect - github.com/envoyproxy/protoc-gen-validate v0.1.0 // indirect - github.com/form3tech-oss/jwt-go v3.2.3+incompatible // indirect - github.com/go-errors/errors v1.0.1 // indirect - github.com/go-kit/log v0.1.0 // indirect - github.com/go-logfmt/logfmt v0.5.0 // indirect - github.com/go-logr/logr v0.4.0 // indirect - github.com/go-ole/go-ole v1.2.4 // indirect - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect - github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.0.1 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/gorilla/websocket v1.4.2 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/jonboulle/clockwork v0.2.2 // indirect - github.com/json-iterator/go v1.1.12 // indirect - github.com/k0kubun/pp v3.0.1+incompatible // indirect - github.com/leodido/go-urn v1.2.1 // indirect - github.com/mattn/go-colorable v0.1.7 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/go-homedir v1.1.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mschoch/smat v0.2.0 // indirect - github.com/pelletier/go-toml v1.7.0 // indirect - github.com/pierrec/lz4 v2.5.2+incompatible // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.32.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect - github.com/prometheus/statsd_exporter v0.21.0 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/shirou/gopsutil v3.20.11+incompatible // indirect - github.com/shirou/gopsutil/v3 v3.21.6 // indirect - github.com/sirupsen/logrus v1.7.0 // indirect - github.com/soheilhy/cmux v0.1.5 // indirect - github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.2.2 // indirect - github.com/spf13/cast v1.3.0 // indirect - github.com/spf13/jwalterweatherman v1.0.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.1 // indirect - github.com/stretchr/objx v0.1.1 // indirect - github.com/subosito/gotenv v1.2.0 // indirect - github.com/tklauser/go-sysconf v0.3.6 // indirect - github.com/tklauser/numcpus v0.2.2 // indirect - github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect - github.com/uber/jaeger-client-go v2.29.1+incompatible // indirect - github.com/uber/jaeger-lib v2.4.1+incompatible // indirect - github.com/ugorji/go/codec v1.2.6 // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.6 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect - go.etcd.io/etcd/client/v2 v2.305.2 // indirect - go.etcd.io/etcd/pkg/v3 v3.5.2 // indirect - go.etcd.io/etcd/raft/v3 v3.5.2 // indirect - go.opencensus.io v0.23.0 // indirect - go.opentelemetry.io/contrib v0.20.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect - go.opentelemetry.io/otel v0.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect - go.opentelemetry.io/otel/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect - go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect - go.opentelemetry.io/otel/trace v0.20.0 // indirect - go.opentelemetry.io/proto/otlp v0.7.0 // indirect - go.uber.org/multierr v1.6.0 // indirect - golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 // indirect - golang.org/x/net v0.0.0-20211105192438-b53810dc28af // indirect - golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect - golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect - golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect - golang.org/x/text v0.3.6 // indirect - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect - google.golang.org/appengine v1.6.6 // indirect - gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.51.0 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect - k8s.io/api v0.16.9 // indirect - k8s.io/klog v1.0.0 // indirect - k8s.io/klog/v2 v2.9.0 // indirect - k8s.io/utils v0.0.0-20190801114015-581e00157fb1 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect -) diff --git a/go.sum b/go.sum index b269da8756..db45e2c152 100644 --- a/go.sum +++ b/go.sum @@ -951,6 +951,7 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5 h1:FR+oGxGfbQu1d+jglI3rCkjAjUnhRSZcUxr+DqlDLNo= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= @@ -965,6 +966,7 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -974,6 +976,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1198,6 +1201,7 @@ golang.org/x/tools v0.0.0-20200928182047-19e03678916f/go.mod h1:z6u4i615ZeAfBE4X golang.org/x/tools v0.0.0-20201014170642-d1624618ad65/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1205,7 +1209,9 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.8.2 h1:CCXrcPKiGGotvnN6jfUsKk4rRqm7q09/YbKb5xCEvtM= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= diff --git a/protocol/invocation.go b/protocol/invocation.go index faf3507ce5..a97cf72a23 100644 --- a/protocol/invocation.go +++ b/protocol/invocation.go @@ -50,6 +50,7 @@ type Invocation interface { GetAttachment(key string) (string, bool) GetAttachmentInterface(string) interface{} GetAttachmentWithDefaultValue(key string, defaultValue string) string + GetAttachmentAsContext() context.Context // Attributes firstly introduced on dubbo-java 2.7.6. It is // used in internal invocation, that is, it's not passed between @@ -58,6 +59,4 @@ type Invocation interface { SetAttribute(key string, value interface{}) GetAttribute(key string) (interface{}, bool) GetAttributeWithDefaultValue(key string, defaultValue interface{}) interface{} - - ToContext() context.Context } diff --git a/protocol/invocation/rpcinvocation.go b/protocol/invocation/rpcinvocation.go index 99dd02124c..f7eae27122 100644 --- a/protocol/invocation/rpcinvocation.go +++ b/protocol/invocation/rpcinvocation.go @@ -244,7 +244,7 @@ func (r *RPCInvocation) GetAttributeWithDefaultValue(key string, defaultValue in return defaultValue } -func (r *RPCInvocation) ToContext() context.Context { +func (r *RPCInvocation) GetAttachmentAsContext() context.Context { gRPCMD := make(metadata.MD, 0) ctx := context.Background() for k, v := range r.Attachments() { diff --git a/registry/xds/registry.go b/registry/xds/registry.go index a0daf8c522..68c9d55bcd 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -51,7 +51,7 @@ type xdsRegistry struct { } func isProvider(url *common.URL) bool { - return getCategory(url) == "providers" + return getCategory(url) == constant.ProviderCategory } func getCategory(url *common.URL) string { diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 9325fb7d7c..2919293505 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -31,7 +31,7 @@ import ( "dubbo.apache.org/dubbo-go/v3/registry" xdsCommon "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" "dubbo.apache.org/dubbo-go/v3/remoting/xds/ewatcher" - "dubbo.apache.org/dubbo-go/v3/remoting/xds/interfaceMapping" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/mapping" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/resource" ) @@ -80,7 +80,7 @@ type WrappedClientImpl struct { /* interfaceMapHandler manages dubbogo metadata containing service key -> hostAddr map */ - interfaceMapHandler interfaceMapping.InterfaceMapHandler + interfaceMapHandler mapping.InterfaceMapHandler /* rdsMap cache router config @@ -168,7 +168,7 @@ func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr xdsCommon } // 4. init interface map handler - newClient.interfaceMapHandler = interfaceMapping.NewInterfaceMapHandlerImpl( + newClient.interfaceMapHandler = mapping.NewInterfaceMapHandlerImpl( newClient.xdsClient, defaultIstiodTokenPath, xdsCommon.NewAddr(newClient.istiodPodIP+":"+defaultIstiodDebugPort), diff --git a/remoting/xds/common/model.go b/remoting/xds/common/model.go index e3532ee19e..04b83946b6 100644 --- a/remoting/xds/common/model.go +++ b/remoting/xds/common/model.go @@ -18,6 +18,7 @@ package common import ( + "net" "strings" ) @@ -27,15 +28,15 @@ type Addr struct { } func NewAddr(addr string) Addr { - addrs := strings.Split(addr, ":") + host, port, _ := net.SplitHostPort(addr) return Addr{ - HostnameOrIP: addrs[0], - Port: addrs[1], + HostnameOrIP: host, + Port: port, } } func (a *Addr) String() string { - return a.HostnameOrIP + ":" + a.Port + return net.JoinHostPort(a.HostnameOrIP, a.Port) } type Cluster struct { diff --git a/remoting/xds/interfaceMapping/debug.go b/remoting/xds/mapping/debug.go similarity index 98% rename from remoting/xds/interfaceMapping/debug.go rename to remoting/xds/mapping/debug.go index b121df5c1f..5606a25b80 100644 --- a/remoting/xds/interfaceMapping/debug.go +++ b/remoting/xds/mapping/debug.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package interfaceMapping +package mapping import ( "encoding/json" diff --git a/remoting/xds/interfaceMapping/debug_test.go b/remoting/xds/mapping/debug_test.go similarity index 99% rename from remoting/xds/interfaceMapping/debug_test.go rename to remoting/xds/mapping/debug_test.go index e1e20ccdd2..32120159bb 100644 --- a/remoting/xds/interfaceMapping/debug_test.go +++ b/remoting/xds/mapping/debug_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package interfaceMapping +package mapping import ( "encoding/json" diff --git a/remoting/xds/interfaceMapping/handler.go b/remoting/xds/mapping/handler.go similarity index 98% rename from remoting/xds/interfaceMapping/handler.go rename to remoting/xds/mapping/handler.go index a92e6d149d..8386c649ab 100644 --- a/remoting/xds/interfaceMapping/handler.go +++ b/remoting/xds/mapping/handler.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package interfaceMapping +package mapping import ( "encoding/json" @@ -82,6 +82,7 @@ func (i *InterfaceMapHandlerImpl) Register(serviceUniqueKey string) error { func (i *InterfaceMapHandlerImpl) GetHostAddrMap(serviceUniqueKey string) (string, error) { i.interfaceNameHostAddrMapLock.RLock() if hostAddr, ok := i.interfaceNameHostAddrMap[serviceUniqueKey]; ok { + i.interfaceNameHostAddrMapLock.RUnlock() return hostAddr, nil } i.interfaceNameHostAddrMapLock.RUnlock() diff --git a/remoting/xds/interfaceMapping/handler_test.go b/remoting/xds/mapping/handler_test.go similarity index 99% rename from remoting/xds/interfaceMapping/handler_test.go rename to remoting/xds/mapping/handler_test.go index feca88dcb1..a6bfdb82fe 100644 --- a/remoting/xds/interfaceMapping/handler_test.go +++ b/remoting/xds/mapping/handler_test.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package interfaceMapping +package mapping import ( "net/http" diff --git a/remoting/xds/interfaceMapping/metadata.go b/remoting/xds/mapping/metadata.go similarity index 98% rename from remoting/xds/interfaceMapping/metadata.go rename to remoting/xds/mapping/metadata.go index 9a44ddd5c0..8a93d61d86 100644 --- a/remoting/xds/interfaceMapping/metadata.go +++ b/remoting/xds/mapping/metadata.go @@ -15,7 +15,7 @@ * limitations under the License. */ -package interfaceMapping +package mapping import ( structpb "github.com/golang/protobuf/ptypes/struct" diff --git a/remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go b/remoting/xds/mapping/mocks/InterfaceMapHandler.go similarity index 100% rename from remoting/xds/interfaceMapping/mocks/InterfaceMapHandler.go rename to remoting/xds/mapping/mocks/InterfaceMapHandler.go diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go index e61c8d5efe..c25ab6a039 100644 --- a/remoting/xds/xds_client_factory.go +++ b/remoting/xds/xds_client_factory.go @@ -26,7 +26,7 @@ import ( import ( xdsCommon "dubbo.apache.org/dubbo-go/v3/remoting/xds/common" - "dubbo.apache.org/dubbo-go/v3/remoting/xds/interfaceMapping" + "dubbo.apache.org/dubbo-go/v3/remoting/xds/mapping" "dubbo.apache.org/dubbo-go/v3/xds/client" "dubbo.apache.org/dubbo-go/v3/xds/client/bootstrap" "dubbo.apache.org/dubbo-go/v3/xds/client/resource/version" @@ -41,7 +41,7 @@ var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAdd UserAgentName: gRPCUserAgentName, UserAgentVersionType: &v3corepb.Node_UserAgentVersion{UserAgentVersion: "1.45.0"}, ClientFeatures: []string{clientFeatureNoOverprovisioning}, - Metadata: interfaceMapping.GetDubboGoMetadata(""), + Metadata: mapping.GetDubboGoMetadata(""), } nonNilCredsConfigV2 := &bootstrap.Config{ From aba60ca82289df770288db1acc868f5ef1d40e60 Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Mon, 4 Apr 2022 14:54:43 +0800 Subject: [PATCH 16/19] Fix: compatiable to go 1.15 --- remoting/xds/mapping/handler.go | 3 +-- remoting/xds/mapping/handler_test.go | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/remoting/xds/mapping/handler.go b/remoting/xds/mapping/handler.go index 8386c649ab..2f8ff84acc 100644 --- a/remoting/xds/mapping/handler.go +++ b/remoting/xds/mapping/handler.go @@ -22,7 +22,6 @@ import ( "fmt" "io/ioutil" "net/http" - "os" "sync" "time" ) @@ -109,7 +108,7 @@ func (i *InterfaceMapHandlerImpl) GetHostAddrMap(serviceUniqueKey string) (strin // 'dubbo-go-app.default.svc.cluster.local:20000' func (i *InterfaceMapHandlerImpl) getServiceUniqueKeyHostAddrMapFromPilot() (map[string]string, error) { req, _ := http.NewRequest(http.MethodGet, fmt.Sprintf("http://%s/debug/adsz", i.istioDebugAddr.String()), nil) - token, err := os.ReadFile(i.istioTokenPath) + token, err := ioutil.ReadFile(i.istioTokenPath) if err != nil { return nil, err } diff --git a/remoting/xds/mapping/handler_test.go b/remoting/xds/mapping/handler_test.go index a6bfdb82fe..d63e2be45e 100644 --- a/remoting/xds/mapping/handler_test.go +++ b/remoting/xds/mapping/handler_test.go @@ -18,8 +18,8 @@ package mapping import ( + "io/ioutil" "net/http" - "os" "testing" "time" ) @@ -125,5 +125,5 @@ func getMatchFunction(metadata string) func(abc *structpb.Struct) bool { } func generateMockToken() error { - return os.WriteFile(istioTokenPathFoo, []byte(istioTokenFoo), 0777) + return ioutil.WriteFile(istioTokenPathFoo, []byte(istioTokenFoo), 0777) } From e41ea182128ceb99b310f97475c81bf9b8240013 Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Mon, 4 Apr 2022 16:27:42 +0800 Subject: [PATCH 17/19] fix: change cluster Id to ID --- .github/workflows/golangci-lint.yml | 2 +- cluster/router/meshrouter/meshrouter.go | 12 ++++++------ common/constant/xds.go | 2 +- registry/directory/directory.go | 10 +++++----- registry/event_test.go | 2 +- remoting/xds/client.go | 2 +- remoting/xds/ewatcher/ewatcher.go | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index ed4dcbc2ef..918325c4e7 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -16,7 +16,7 @@ jobs: steps: - uses: actions/setup-go@v2 with: - go-version: 1.17 + go-version: 1.15 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3.1.0 diff --git a/cluster/router/meshrouter/meshrouter.go b/cluster/router/meshrouter/meshrouter.go index a39376d1b3..102a48ac7d 100644 --- a/cluster/router/meshrouter/meshrouter.go +++ b/cluster/router/meshrouter/meshrouter.go @@ -69,11 +69,11 @@ func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocat clusterInvokerMap := make(map[string][]protocol.Invoker) for _, v := range invokers { - meshClusterId := v.GetURL().GetParam(constant.MeshClusterIDKey, "") - if _, ok := clusterInvokerMap[meshClusterId]; !ok { - clusterInvokerMap[meshClusterId] = make([]protocol.Invoker, 0) + meshClusterID := v.GetURL().GetParam(constant.MeshClusterIDKey, "") + if _, ok := clusterInvokerMap[meshClusterID]; !ok { + clusterInvokerMap[meshClusterID] = make([]protocol.Invoker, 0) } - clusterInvokerMap[meshClusterId] = append(clusterInvokerMap[meshClusterId], v) + clusterInvokerMap[meshClusterID] = append(clusterInvokerMap[meshClusterID], v) } if len(rconf.VirtualHosts) != 0 { @@ -102,9 +102,9 @@ func (r *MeshRouter) Route(invokers []protocol.Invoker, url *common.URL, invocat } invokersWeightPairs := make(invokerWeightPairs, 0) - for clusterId, weight := range r.WeightedClusters { + for clusterID, weight := range r.WeightedClusters { // cluster -> invokers - targetInvokers := clusterInvokerMap[clusterId] + targetInvokers := clusterInvokerMap[clusterID] invokersWeightPairs = append(invokersWeightPairs, invokerWeightPair{ invokers: targetInvokers, weight: weight.Weight, diff --git a/common/constant/xds.go b/common/constant/xds.go index 53a0f8c2e0..5c92d0ab4d 100644 --- a/common/constant/xds.go +++ b/common/constant/xds.go @@ -18,7 +18,7 @@ package constant const ( - MeshClusterIDKey = "meshClusterId" + MeshClusterIDKey = "meshClusterID" MeshHostAddrKey = "meshHostAddr" MeshSubsetKey = "meshSubset" diff --git a/registry/directory/directory.go b/registry/directory/directory.go index d4d68626ec..47337954e1 100644 --- a/registry/directory/directory.go +++ b/registry/directory/directory.go @@ -314,11 +314,11 @@ func (dir *RegistryDirectory) toGroupInvokers() []protocol.Invoker { return groupInvokersList } -func (dir *RegistryDirectory) uncacheInvokerWithClusterId(clusterId string) []protocol.Invoker { - logger.Debugf("All service will be deleted in cache invokers with clusterId %s!", clusterId) +func (dir *RegistryDirectory) uncacheInvokerWithClusterID(clusterID string) []protocol.Invoker { + logger.Debugf("All service will be deleted in cache invokers with clusterID %s!", clusterID) invokerKeys := make([]string, 0) dir.cacheInvokersMap.Range(func(key, cacheInvoker interface{}) bool { - if cacheInvoker.(protocol.Invoker).GetURL().GetParam(constant.MeshClusterIDKey, "") == clusterId { + if cacheInvoker.(protocol.Invoker).GetURL().GetParam(constant.MeshClusterIDKey, "") == clusterID { invokerKeys = append(invokerKeys, key.(string)) } return true @@ -332,8 +332,8 @@ func (dir *RegistryDirectory) uncacheInvokerWithClusterId(clusterId string) []pr // uncacheInvoker will return abandoned Invoker, if no Invoker to be abandoned, return nil func (dir *RegistryDirectory) uncacheInvoker(event *registry.ServiceEvent) []protocol.Invoker { - if clusterId := event.Service.GetParam(constant.MeshClusterIDKey, ""); event.Service.Location == constant.MeshAnyAddrMatcher && clusterId != "" { - dir.uncacheInvokerWithClusterId(clusterId) + if clusterID := event.Service.GetParam(constant.MeshClusterIDKey, ""); event.Service.Location == constant.MeshAnyAddrMatcher && clusterID != "" { + dir.uncacheInvokerWithClusterID(clusterID) } return []protocol.Invoker{dir.uncacheInvokerWithKey(event.Key())} } diff --git a/registry/event_test.go b/registry/event_test.go index e772056b30..50d3cebb25 100644 --- a/registry/event_test.go +++ b/registry/event_test.go @@ -34,7 +34,7 @@ func TestKey(t *testing.T) { se := ServiceEvent{ Service: u1, } - assert.Equal(t, se.Key(), "dubbo://:@127.0.0.1:20000/?interface=com.ikurento.user.UserProvider&group=&version=2.0×tamp=&meshClusterId=") + assert.Equal(t, se.Key(), "dubbo://:@127.0.0.1:20000/?interface=com.ikurento.user.UserProvider&group=&version=2.0×tamp=&meshClusterID=") se2 := ServiceEvent{ Service: u1, diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 2919293505..6f0fb64a2c 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -90,7 +90,7 @@ type WrappedClientImpl struct { rdsMapLock sync.RWMutex /* - cdsMap cache full clusterId -> clusterUpdate map of this istiod + cdsMap cache full clusterID -> clusterUpdate map of this istiod */ cdsMap map[string]resource.ClusterUpdate cdsMapLock sync.RWMutex diff --git a/remoting/xds/ewatcher/ewatcher.go b/remoting/xds/ewatcher/ewatcher.go index fabda71831..06a2a4f202 100644 --- a/remoting/xds/ewatcher/ewatcher.go +++ b/remoting/xds/ewatcher/ewatcher.go @@ -87,7 +87,7 @@ func (watcher *endPointWatcherCtx) Destroy() { watcher.cancel() } /* - directory would identify this by EndpointHealthStatusUnhealthy and Location == "*" and none empty clusterId + directory would identify this by EndpointHealthStatusUnhealthy and Location == "*" and none empty clusterID and delete related invokers */ event := generateRegistryEvent(watcher.clusterName, resource.Endpoint{ From 53a86dde9561ed653f7d59623c61f59174478d1f Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Tue, 5 Apr 2022 15:25:23 +0800 Subject: [PATCH 18/19] fix: rename to host addr --- registry/xds/registry.go | 2 +- remoting/xds/client.go | 22 +++++----- remoting/xds/client_test.go | 28 ++++++------- remoting/xds/common/addr.go | 43 ++++++++++++++++++++ remoting/xds/common/{model.go => cluster.go} | 26 ++---------- remoting/xds/mapping/handler.go | 6 +-- remoting/xds/mapping/handler_test.go | 6 +-- remoting/xds/xds_client_factory.go | 2 +- 8 files changed, 79 insertions(+), 56 deletions(-) create mode 100644 remoting/xds/common/addr.go rename remoting/xds/common/{model.go => cluster.go} (76%) diff --git a/registry/xds/registry.go b/registry/xds/registry.go index 68c9d55bcd..0ec40eda2d 100644 --- a/registry/xds/registry.go +++ b/registry/xds/registry.go @@ -163,7 +163,7 @@ func newXDSRegistry(url *common.URL) (registry.Registry, error) { constant.PodNameEnvKey, constant.PodNamespaceEnvKey) } - wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, common2.NewAddr(url.Ip+":"+url.Port)) + wrappedXDSClient, err := xds.NewXDSWrappedClient(pn, ns, localIP, common2.NewHostNameOrIPAddr(url.Ip+":"+url.Port)) if err != nil { return nil, err } diff --git a/remoting/xds/client.go b/remoting/xds/client.go index 6f0fb64a2c..7bfec7c5e8 100644 --- a/remoting/xds/client.go +++ b/remoting/xds/client.go @@ -62,14 +62,14 @@ type WrappedClientImpl struct { /* hostAddr is local pod's cluster and hostAddr, like dubbo-go-app.default.svc.cluster.local:20000 */ - hostAddr xdsCommon.Addr + hostAddr xdsCommon.HostAddr /* istiod info istiodAddr is istio $(istioSeviceFullName):$(xds-grpc-port) like istiod.istio-system.svc.cluster.local:15010 istiodPodIP is to call istiod unexposed debug port 8080 */ - istiodAddr xdsCommon.Addr + istiodAddr xdsCommon.HostAddr istiodPodIP string /* @@ -131,7 +131,7 @@ func GetXDSWrappedClient() *WrappedClientImpl { } // NewXDSWrappedClient create or get singleton xdsWrappedClient -func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr xdsCommon.Addr) (XDSWrapperClient, error) { +func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr xdsCommon.HostAddr) (XDSWrapperClient, error) { // todo @(laurence) safety problem? what if to concurrent 'new' both create new client? if xdsWrappedClient != nil { return xdsWrappedClient, nil @@ -171,7 +171,7 @@ func NewXDSWrappedClient(podName, namespace, localIP string, istioAddr xdsCommon newClient.interfaceMapHandler = mapping.NewInterfaceMapHandlerImpl( newClient.xdsClient, defaultIstiodTokenPath, - xdsCommon.NewAddr(newClient.istiodPodIP+":"+defaultIstiodDebugPort), + xdsCommon.NewHostNameOrIPAddr(newClient.istiodPodIP+":"+defaultIstiodDebugPort), newClient.hostAddr) xdsWrappedClient = newClient @@ -202,7 +202,7 @@ func (w *WrappedClientImpl) GetRouterConfig(hostAddr string) resource.RouteConfi } func (w *WrappedClientImpl) GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate { - addr := xdsCommon.NewAddr(hostAddr) + addr := xdsCommon.NewHostNameOrIPAddr(hostAddr) w.cdsMapLock.RLock() defer w.cdsMapLock.Unlock() for clusterName, v := range w.cdsMap { @@ -234,7 +234,7 @@ func (w *WrappedClientImpl) UnSubscribe(svcUniqueName string) { w.subscribeStopChMap.Delete(svcUniqueName) } -func (w *WrappedClientImpl) GetHostAddress() xdsCommon.Addr { +func (w *WrappedClientImpl) GetHostAddress() xdsCommon.HostAddr { return w.hostAddr } @@ -252,7 +252,7 @@ func (w *WrappedClientImpl) registerHostLevelSubscription(hostAddr, interfaceNam w.hostAddrListenerMapLock.Unlock() return } - // host Addr key must not exist in map, create one + // host HostAddr key must not exist in map, create one w.hostAddrListenerMap[hostAddr] = make(map[string]registry.NotifyListener) w.hostAddrClusterCtxMapLock.Lock() @@ -412,7 +412,7 @@ func (w *WrappedClientImpl) startWatchingAllClusterAndLoadLocalHostAddrAndIstioP } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - w.istiodPodIP = xdsCommon.NewAddr(e.Address).HostnameOrIP + w.istiodPodIP = xdsCommon.NewHostNameOrIPAddr(e.Address).HostnameOrIP foundIstiod = true close(foundLocalStopCh) } @@ -428,7 +428,7 @@ func (w *WrappedClientImpl) startWatchingAllClusterAndLoadLocalHostAddrAndIstioP } for _, v := range endpoint.Localities { for _, e := range v.Endpoints { - if xdsCommon.NewAddr(e.Address).HostnameOrIP == w.localIP { + if xdsCommon.NewHostNameOrIPAddr(e.Address).HostnameOrIP == w.localIP { cluster := xdsCommon.NewCluster(update.ClusterName) w.hostAddr = cluster.Addr foundLocal = true @@ -485,7 +485,7 @@ func (w *WrappedClientImpl) runWatchingCdsUpdateEvent() { // 'outbound|20000||dubbo-go-app.default.svc.cluster.local', // 'outbound|20000|v2|dubbo-go-app.default.svc.cluster.local'] func (w *WrappedClientImpl) getAllVersionClusterName(hostAddr string) []string { - addr := xdsCommon.NewAddr(hostAddr) + addr := xdsCommon.NewHostNameOrIPAddr(hostAddr) allVersionClusterNames := make([]string, 0) w.cdsMapLock.RLock() defer w.cdsMapLock.RUnlock() @@ -505,6 +505,6 @@ type XDSWrapperClient interface { GetHostAddrByServiceUniqueKey(serviceUniqueKey string) (string, error) ChangeInterfaceMap(serviceUniqueKey string, add bool) error GetClusterUpdateIgnoreVersion(hostAddr string) resource.ClusterUpdate - GetHostAddress() xdsCommon.Addr + GetHostAddress() xdsCommon.HostAddr GetIstioPodIP() string } diff --git a/remoting/xds/client_test.go b/remoting/xds/client_test.go index 60fc5e86d1..254fd5f02c 100644 --- a/remoting/xds/client_test.go +++ b/remoting/xds/client_test.go @@ -165,10 +165,10 @@ func testWithDiscoverySuccess(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Nil(t, err) assert.NotNil(t, xdsWrappedClient) @@ -246,10 +246,10 @@ func testFailedWithIstioCDS(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Equal(t, DiscoverIstioPodError, err) assert.Nil(t, xdsWrappedClient) assert.Equal(t, int32(1), cancelCalledCounter.Load()) @@ -321,10 +321,10 @@ func testFailedWithLocalCDS(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Equal(t, DiscoverLocalError, err) assert.Nil(t, xdsWrappedClient) assert.Equal(t, int32(1), cancelCalledCounter.Load()) @@ -396,10 +396,10 @@ func testFailedWithNoneCDS(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Equal(t, DiscoverIstioPodError, err) assert.Nil(t, xdsWrappedClient) assert.Equal(t, int32(0), cancelCalledCounter.Load()) @@ -471,10 +471,10 @@ func testFailedWithLocalEDSFailed(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Equal(t, DiscoverLocalError, err) assert.Nil(t, xdsWrappedClient) assert.Equal(t, int32(2), cancelCalledCounter.Load()) @@ -546,10 +546,10 @@ func testFailedWithIstioEDSFailed(t *testing.T) { cancelCalledCounter.Inc() }) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Equal(t, DiscoverIstioPodError, err) assert.Nil(t, xdsWrappedClient) assert.Equal(t, int32(2), cancelCalledCounter.Load()) @@ -717,12 +717,12 @@ func testSubscribe(t *testing.T) { })). Return(func() {}) - xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.Addr) (client.XDSClient, error) { + xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr common.HostAddr) (client.XDSClient, error) { return mockXDSClient, nil } xdsWrappedClient = nil - xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewAddr(istioHostAddrFoo)) + xdsWrappedClient, err := NewXDSWrappedClient(podNameFoo, localNamespaceFoo, localIPFoo, common.NewHostNameOrIPAddr(istioHostAddrFoo)) assert.Nil(t, err) assert.NotNil(t, xdsWrappedClient) diff --git a/remoting/xds/common/addr.go b/remoting/xds/common/addr.go new file mode 100644 index 0000000000..9cd504ab2b --- /dev/null +++ b/remoting/xds/common/addr.go @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package common + +import ( + "net" +) + +type HostAddr struct { + HostnameOrIP string + Port string +} + +func NewHostNameOrIPAddr(addr string) HostAddr { + host, port, _ := net.SplitHostPort(addr) + return HostAddr{ + HostnameOrIP: host, + Port: port, + } +} + +func (a *HostAddr) Network() string { + return "tcp" +} + +func (a *HostAddr) String() string { + return net.JoinHostPort(a.HostnameOrIP, a.Port) +} diff --git a/remoting/xds/common/model.go b/remoting/xds/common/cluster.go similarity index 76% rename from remoting/xds/common/model.go rename to remoting/xds/common/cluster.go index 04b83946b6..19713438dd 100644 --- a/remoting/xds/common/model.go +++ b/remoting/xds/common/cluster.go @@ -17,31 +17,11 @@ package common -import ( - "net" - "strings" -) - -type Addr struct { - HostnameOrIP string - Port string -} - -func NewAddr(addr string) Addr { - host, port, _ := net.SplitHostPort(addr) - return Addr{ - HostnameOrIP: host, - Port: port, - } -} - -func (a *Addr) String() string { - return net.JoinHostPort(a.HostnameOrIP, a.Port) -} +import "strings" type Cluster struct { Bound string - Addr Addr + Addr HostAddr Subset string } @@ -49,7 +29,7 @@ func NewCluster(clusterID string) Cluster { clusterIDs := strings.Split(clusterID, "|") return Cluster{ Bound: clusterIDs[0], - Addr: Addr{ + Addr: HostAddr{ Port: clusterIDs[1], HostnameOrIP: clusterIDs[3], }, diff --git a/remoting/xds/mapping/handler.go b/remoting/xds/mapping/handler.go index 2f8ff84acc..b204317874 100644 --- a/remoting/xds/mapping/handler.go +++ b/remoting/xds/mapping/handler.go @@ -42,9 +42,9 @@ const ( ) type InterfaceMapHandlerImpl struct { - hostAddr common.Addr + hostAddr common.HostAddr - istioDebugAddr common.Addr + istioDebugAddr common.HostAddr xdsClient client.XDSClient @@ -138,7 +138,7 @@ func (i *InterfaceMapHandlerImpl) interfaceAppNameMap2DubboGoMetadata() *structp return GetDubboGoMetadata(string(data)) } -func NewInterfaceMapHandlerImpl(xdsClient client.XDSClient, istioTokenPath string, istioDebugAddr, hostAddr common.Addr) InterfaceMapHandler { +func NewInterfaceMapHandlerImpl(xdsClient client.XDSClient, istioTokenPath string, istioDebugAddr, hostAddr common.HostAddr) InterfaceMapHandler { return &InterfaceMapHandlerImpl{ xdsClient: xdsClient, interfaceAppNameMap: map[string]string{}, diff --git a/remoting/xds/mapping/handler_test.go b/remoting/xds/mapping/handler_test.go index d63e2be45e..2db03511fe 100644 --- a/remoting/xds/mapping/handler_test.go +++ b/remoting/xds/mapping/handler_test.go @@ -53,7 +53,7 @@ const ( func TestNewInterfaceMapHandler(t *testing.T) { mockXDSClient := &mocks.XDSClient{} - interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewHostNameOrIPAddr(istiodDebugAddrStrFoo), common.NewHostNameOrIPAddr(localPodServiceAddr)) assert.NotNil(t, interfaceMapHandler) } @@ -61,7 +61,7 @@ func TestInterfaceMapHandlerRegisterAndUnregister(t *testing.T) { mockXDSClient := &mocks.XDSClient{} mockXDSClient.On("SetMetadata", mock.AnythingOfType("*structpb.Struct")).Return(nil) - interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewHostNameOrIPAddr(istiodDebugAddrStrFoo), common.NewHostNameOrIPAddr(localPodServiceAddr)) assert.Nil(t, interfaceMapHandler.Register(serviceKey1)) assert.Nil(t, interfaceMapHandler.Register(serviceKey2)) @@ -76,7 +76,7 @@ func TestInterfaceMapHandlerRegisterAndUnregister(t *testing.T) { func TestGetServiceUniqueKeyHostAddrMapFromPilot(t *testing.T) { mockXDSClient := &mocks.XDSClient{} - interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewAddr(istiodDebugAddrStrFoo), common.NewAddr(localPodServiceAddr)) + interfaceMapHandler := NewInterfaceMapHandlerImpl(mockXDSClient, istioTokenPathFoo, common.NewHostNameOrIPAddr(istiodDebugAddrStrFoo), common.NewHostNameOrIPAddr(localPodServiceAddr)) assert.Nil(t, generateMockToken()) // 1. start mock http server diff --git a/remoting/xds/xds_client_factory.go b/remoting/xds/xds_client_factory.go index c25ab6a039..f038e6467a 100644 --- a/remoting/xds/xds_client_factory.go +++ b/remoting/xds/xds_client_factory.go @@ -34,7 +34,7 @@ import ( // xdsClientFactoryFunction generates new xds client // when running ut, it's for for ut to replace -var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr xdsCommon.Addr) (client.XDSClient, error) { +var xdsClientFactoryFunction = func(localIP, podName, namespace string, istioAddr xdsCommon.HostAddr) (client.XDSClient, error) { // todo fix these ugly magic num v3NodeProto := &v3corepb.Node{ Id: "sidecar~" + localIP + "~" + podName + "." + namespace + "~" + namespace + ".svc.cluster.local", From abd851a51ce416252f16eb40cc126abdfd8b1af6 Mon Sep 17 00:00:00 2001 From: LaurenceLiZhixin <382673304@qq.com> Date: Tue, 5 Apr 2022 20:06:47 +0800 Subject: [PATCH 19/19] Fix: import formatter --- remoting/xds/common/cluster.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/remoting/xds/common/cluster.go b/remoting/xds/common/cluster.go index 19713438dd..a2daf15640 100644 --- a/remoting/xds/common/cluster.go +++ b/remoting/xds/common/cluster.go @@ -17,7 +17,9 @@ package common -import "strings" +import ( + "strings" +) type Cluster struct { Bound string