Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use dynamicMetadata to construct metadataMatchCriteria() #3254

Merged
merged 9 commits into from
May 4, 2018
Merged

Use dynamicMetadata to construct metadataMatchCriteria() #3254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bde0c9c
Use dynamicMetadata to construct metadataMatchCriteria()
Apr 30, 2018
1747131
Fix formatting issues
Apr 30, 2018
be4fe35
Fix more tests
Apr 30, 2018
f1f12e0
Add a test for overriding metadata matcha via dynamic metadata
Apr 30, 2018
b261a23
Handle route_entry_->metadataMatchCriteria() == nullptr
May 1, 2018
663c922
Fix formatting
May 1, 2018
b24da82
Improve method docs.
May 3, 2018
d29a4a5
Return the cached result of metadataMatchCriteria()
May 3, 2018
8b7bb04
More tests
May 3, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions include/envoy/router/router.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,9 @@ class MetadataMatchCriterion {

typedef std::shared_ptr<const MetadataMatchCriterion> MetadataMatchCriterionConstSharedPtr;

class MetadataMatchCriteria;
typedef std::unique_ptr<const MetadataMatchCriteria> MetadataMatchCriteriaConstPtr;

class MetadataMatchCriteria {
public:
virtual ~MetadataMatchCriteria() {}
Expand All @@ -345,6 +348,17 @@ class MetadataMatchCriteria {
*/
virtual const std::vector<MetadataMatchCriterionConstSharedPtr>&
metadataMatchCriteria() const PURE;

/**
* Creates a new MetadataMatchCriteria, merging existing
* metadata criteria with the provided criteria. The result criteria is the
* combination of both sets of criteria, with those from the metadata_matches
* ProtobufWkt::Struct taking precedence.
* @param metadata_matches supplies the new criteria.
* @return MetadataMatchCriteriaConstPtr the result criteria.
*/
virtual MetadataMatchCriteriaConstPtr
mergeMatchCriteria(const ProtobufWkt::Struct& metadata_matches) const PURE;
};

/**
Expand Down
2 changes: 1 addition & 1 deletion include/envoy/upstream/load_balancer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ class LoadBalancerContext {
* @return Router::MetadataMatchCriteria* metadata for use in selecting a subset of hosts
* during load balancing.
*/
virtual const Router::MetadataMatchCriteria* metadataMatchCriteria() const PURE;
virtual const Router::MetadataMatchCriteria* metadataMatchCriteria() PURE;

/**
* @return const Network::Connection* the incoming connection or nullptr to use during load
Expand Down
2 changes: 1 addition & 1 deletion source/common/http/websocket/ws_handler_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ class WsHandlerImpl : public Extensions::NetworkFilters::TcpProxy::TcpProxyFilte
Network::ReadFilterCallbacks* read_callbacks);

// Upstream::LoadBalancerContext
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override {
const Router::MetadataMatchCriteria* metadataMatchCriteria() override {
return route_entry_.metadataMatchCriteria();
}

Expand Down
14 changes: 4 additions & 10 deletions source/common/router/config_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,14 +256,8 @@ class MetadataMatchCriteriaImpl : public MetadataMatchCriteria {
MetadataMatchCriteriaImpl(const ProtobufWkt::Struct& metadata_matches)
: metadata_match_criteria_(extractMetadataMatchCriteria(nullptr, metadata_matches)){};

/**
* Creates a new MetadataMatchCriteriaImpl, merging existing
* metadata criteria this criteria. The result criteria is the
* combination of both sets of criteria, with those from the
* ProtobufWkt::Struct taking precedence.
*/
MetadataMatchCriteriaImplConstPtr
mergeMatchCriteria(const ProtobufWkt::Struct& metadata_matches) const {
MetadataMatchCriteriaConstPtr
mergeMatchCriteria(const ProtobufWkt::Struct& metadata_matches) const override {
return MetadataMatchCriteriaImplConstPtr(
new MetadataMatchCriteriaImpl(extractMetadataMatchCriteria(this, metadata_matches)));
}
Expand Down Expand Up @@ -495,7 +489,7 @@ class RouteEntryImplBase : public RouteEntry,
const std::string runtime_key_;
Runtime::Loader& loader_;
const uint64_t cluster_weight_;
MetadataMatchCriteriaImplConstPtr cluster_metadata_match_criteria_;
MetadataMatchCriteriaConstPtr cluster_metadata_match_criteria_;
HeaderParserPtr request_headers_parser_;
HeaderParserPtr response_headers_parser_;
PerFilterConfigs per_filter_configs_;
Expand Down Expand Up @@ -539,7 +533,7 @@ class RouteEntryImplBase : public RouteEntry,
std::vector<WeightedClusterEntrySharedPtr> weighted_clusters_;
const uint64_t total_cluster_weight_;
std::unique_ptr<const HashPolicyImpl> hash_policy_;
MetadataMatchCriteriaImplConstPtr metadata_match_criteria_;
MetadataMatchCriteriaConstPtr metadata_match_criteria_;
HeaderParserPtr request_headers_parser_;
HeaderParserPtr response_headers_parser_;
envoy::api::v2::core::Metadata metadata_;
Expand Down
24 changes: 23 additions & 1 deletion source/common/router/router.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,10 @@
#include "common/common/hash.h"
#include "common/common/hex.h"
#include "common/common/logger.h"
#include "common/config/well_known_names.h"
#include "common/http/utility.h"
#include "common/request_info/request_info_impl.h"
#include "common/router/config_impl.h"

namespace Envoy {
namespace Router {
Expand Down Expand Up @@ -161,8 +163,27 @@ class Filter : Logger::Loggable<Logger::Id::router>,
}
return {};
}
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override {
const Router::MetadataMatchCriteria* metadataMatchCriteria() override {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the retry case you may end up calling this method multiple times. Is it worth checking to see if metadata_match_ is not nullptr and returning if not? Is there any chance that doing the merge multiple times could have strange effects? If not to both of these, I would still add a comment about how this can happen multiple times for a single request.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think checking for nullptr and returning early is worth it. Multiple calls to merge should yield the same result, but we can avoid the computation and creating new objects.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mattklein123 hold on -- can the route_entry_ for a request change (e.g.: someone called clearRouteCache() and then metadataMatchCriteria() gets called again)? In that case, caching the result for metadataMatchCriteria() would be wrong... we'd want to recompute.

cc: @zuercher @htuch

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think it's ok, as long as metadataMatchCriteria() isn't invoked until LoadBalancer::chooseHost, which happens during Router::Filter::getConnPool in Filter::decodeHeaders and again in Filter::doRetry. The route_ and route_entry_ get set in decodeHeaders before load balancing, and I believe that once that happens they do not change.

clearRouteCache operates on the Http::ConnectionManagerImpl::ActiveStreamFilterBase which ends up being the source of the route_ and therefore route_entry_ via its implementation of StreamFilterCallbacks::route().

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems right, can we add an ASSERT to this effect (i.e. if we get called twice, the route entry is the same)?

if (route_entry_) {
// Have we been called before? If so, there's no need to recompute because
// by the time this method is called for the first time, route_entry_ should
// not change anymore.
if (metadata_match_ != nullptr) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a test for this as well? Thanks.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@htuch done — plus two bonus tests:

  • using > 1 key/value pair in dynamicMetadata()
  • mixing a key that overrides the route entry with one that doesn't

thanks for the review!

return metadata_match_.get();
}

// The request's metadata, if present, takes precedence over the route's.
const auto& request_metadata = callbacks_->requestInfo().dynamicMetadata().filter_metadata();
const auto filter_it = request_metadata.find(Envoy::Config::MetadataFilters::get().ENVOY_LB);
if (filter_it != request_metadata.end()) {
if (route_entry_->metadataMatchCriteria() != nullptr) {
metadata_match_ =
route_entry_->metadataMatchCriteria()->mergeMatchCriteria(filter_it->second);
} else {
metadata_match_ = std::make_unique<Router::MetadataMatchCriteriaImpl>(filter_it->second);
}
return metadata_match_.get();
}
return route_entry_->metadataMatchCriteria();
}
return nullptr;
Expand Down Expand Up @@ -343,6 +364,7 @@ class Filter : Logger::Loggable<Logger::Id::router>,
MonotonicTime downstream_request_complete_time_;
uint32_t buffer_limit_{0};
bool stream_destroyed_{};
MetadataMatchCriteriaConstPtr metadata_match_;

// list of cookies to add to upstream headers
std::vector<std::string> downstream_set_cookies_;
Expand Down
2 changes: 1 addition & 1 deletion source/extensions/filters/network/redis_proxy/conn_pool_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ class InstanceImpl : public Instance {
// TODO(danielhochman): convert to HashUtil::xxHash64 when we have a migration strategy.
// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return hash_key_; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }
const Network::Connection* downstreamConnection() const override { return nullptr; }

const absl::optional<uint64_t> hash_key_;
Expand Down
2 changes: 1 addition & 1 deletion source/extensions/filters/network/tcp_proxy/tcp_proxy.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ class TcpProxyFilter : public Network::ReadFilter,

// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return {}; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override {
const Router::MetadataMatchCriteria* metadataMatchCriteria() override {
if (config_) {
return config_->metadataMatchCriteria();
}
Expand Down
2 changes: 1 addition & 1 deletion test/common/router/config_impl_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3622,7 +3622,7 @@ TEST(MetadataMatchCriteriaImpl, Merge) {
mutable_fields->insert({"b++", v2});
mutable_fields->insert({"c", v3});

MetadataMatchCriteriaImplConstPtr matches = parent_matches.mergeMatchCriteria(metadata_struct);
MetadataMatchCriteriaConstPtr matches = parent_matches.mergeMatchCriteria(metadata_struct);

EXPECT_EQ(matches->metadataMatchCriteria().size(), 4);
auto it = matches->metadataMatchCriteria().begin();
Expand Down
79 changes: 77 additions & 2 deletions test/common/router/router_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@

#include "common/buffer/buffer_impl.h"
#include "common/common/empty_string.h"
#include "common/config/metadata.h"
#include "common/config/well_known_names.h"
#include "common/network/utility.h"
#include "common/router/config_impl.h"
#include "common/router/router.h"
#include "common/tracing/http_tracer_impl.h"
#include "common/upstream/upstream_impl.h"
Expand Down Expand Up @@ -104,6 +107,72 @@ class RouterTestBase : public testing::Test {
return AssertionSuccess();
}

void verifyMetadataMatchCriteriaFromRequest(bool route_entry_has_match) {
ProtobufWkt::Struct request_struct, route_struct;
ProtobufWkt::Value val;

// Populate metadata like RequestInfo.setDynamicMetadata() would.
auto& fields_map = *request_struct.mutable_fields();
val.set_string_value("v3.1");
fields_map["version"] = val;
val.set_string_value("devel");
fields_map["stage"] = val;
(*callbacks_.request_info_.metadata_
.mutable_filter_metadata())[Envoy::Config::MetadataFilters::get().ENVOY_LB] =
request_struct;

// Populate route entry's metadata which will be overridden.
val.set_string_value("v3.0");
fields_map = *request_struct.mutable_fields();
fields_map["version"] = val;
MetadataMatchCriteriaImpl route_entry_matches(route_struct);

if (route_entry_has_match) {
ON_CALL(callbacks_.route_->route_entry_, metadataMatchCriteria())
.WillByDefault(Return(&route_entry_matches));
} else {
ON_CALL(callbacks_.route_->route_entry_, metadataMatchCriteria())
.WillByDefault(Return(nullptr));
}

EXPECT_CALL(cm_, httpConnPoolForCluster(_, _, _, _))
.WillOnce(
Invoke([&](const std::string&, Upstream::ResourcePriority, Http::Protocol,
Upstream::LoadBalancerContext* context) -> Http::ConnectionPool::Instance* {
auto match = context->metadataMatchCriteria()->metadataMatchCriteria();
EXPECT_EQ(match.size(), 2);
auto it = match.begin();

// Note: metadataMatchCriteria() keeps its entries sorted, so the order for checks
// below matters.

// `stage` was only set by the request, not by the route entry.
EXPECT_EQ((*it)->name(), "stage");
EXPECT_EQ((*it)->value().value().string_value(), "devel");
it++;

// `version` should be what came from the request, overriding the route entry.
EXPECT_EQ((*it)->name(), "version");
EXPECT_EQ((*it)->value().value().string_value(), "v3.1");

// When metadataMatchCriteria() is computed from dynamic metadata, the result should
// be cached.
EXPECT_EQ(context->metadataMatchCriteria(), context->metadataMatchCriteria());

return &cm_.conn_pool_;
}));
EXPECT_CALL(cm_.conn_pool_, newStream(_, _)).WillOnce(Return(&cancellable_));
expectResponseTimerCreate();

Http::TestHeaderMapImpl headers;
HttpTestUtility::addDefaultHeaders(headers);
router_.decodeHeaders(headers, true);

// When the router filter gets reset we should cancel the pool request.
EXPECT_CALL(cancellable_, cancel());
router_.onDestroy();
}

std::string upstream_zone_{"to_az"};
envoy::api::v2::core::Locality upstream_locality_;
Stats::IsolatedStoreImpl stats_store_;
Expand Down Expand Up @@ -466,8 +535,6 @@ TEST_F(RouterTest, AddMultipleCookies) {
TEST_F(RouterTest, MetadataNoOp) { EXPECT_EQ(nullptr, router_.metadataMatchCriteria()); }

TEST_F(RouterTest, MetadataMatchCriteria) {
MockMetadataMatchCriteria matches;

ON_CALL(callbacks_.route_->route_entry_, metadataMatchCriteria())
.WillByDefault(Return(&callbacks_.route_->route_entry_.metadata_matches_criteria_));
EXPECT_CALL(cm_, httpConnPoolForCluster(_, _, _, _))
Expand All @@ -490,6 +557,14 @@ TEST_F(RouterTest, MetadataMatchCriteria) {
router_.onDestroy();
}

TEST_F(RouterTest, MetadataMatchCriteriaFromRequest) {
verifyMetadataMatchCriteriaFromRequest(true);
}

TEST_F(RouterTest, MetadataMatchCriteriaFromRequestNoRouteEntryMatch) {
verifyMetadataMatchCriteriaFromRequest(false);
}

TEST_F(RouterTest, NoMetadataMatchCriteria) {
ON_CALL(callbacks_.route_->route_entry_, metadataMatchCriteria()).WillByDefault(Return(nullptr));
EXPECT_CALL(cm_, httpConnPoolForCluster(_, _, _, _))
Expand Down
2 changes: 1 addition & 1 deletion test/common/upstream/load_balancer_benchmark.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ class TestLoadBalancerContext : public LoadBalancerContext {
public:
// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return hash_key_; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }
const Network::Connection* downstreamConnection() const override { return nullptr; }

absl::optional<uint64_t> hash_key_;
Expand Down
2 changes: 1 addition & 1 deletion test/common/upstream/maglev_lb_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class TestLoadBalancerContext : public LoadBalancerContext {

// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return hash_key_; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }
const Network::Connection* downstreamConnection() const override { return nullptr; }

absl::optional<uint64_t> hash_key_;
Expand Down
2 changes: 1 addition & 1 deletion test/common/upstream/original_dst_cluster_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ class TestLoadBalancerContext : public LoadBalancerContext {
// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return 0; }
const Network::Connection* downstreamConnection() const override { return connection_; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }

absl::optional<uint64_t> hash_key_;
const Network::Connection* connection_;
Expand Down
2 changes: 1 addition & 1 deletion test/common/upstream/ring_hash_lb_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ class TestLoadBalancerContext : public LoadBalancerContext {

// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return hash_key_; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return nullptr; }
const Network::Connection* downstreamConnection() const override { return nullptr; }

absl::optional<uint64_t> hash_key_;
Expand Down
9 changes: 6 additions & 3 deletions test/common/upstream/subset_lb_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,11 @@ class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria {
return matches_;
}

Router::MetadataMatchCriteriaConstPtr
mergeMatchCriteria(const ProtobufWkt::Struct&) const override {
return nullptr;
}

private:
std::vector<Router::MetadataMatchCriterionConstSharedPtr> matches_;
};
Expand All @@ -92,9 +97,7 @@ class TestLoadBalancerContext : public LoadBalancerContext {
// Upstream::LoadBalancerContext
absl::optional<uint64_t> computeHashKey() override { return {}; }
const Network::Connection* downstreamConnection() const override { return nullptr; }
const Router::MetadataMatchCriteria* metadataMatchCriteria() const override {
return matches_.get();
}
const Router::MetadataMatchCriteria* metadataMatchCriteria() override { return matches_.get(); }

private:
const std::shared_ptr<Router::MetadataMatchCriteria> matches_;
Expand Down
1 change: 1 addition & 0 deletions test/mocks/request_info/mocks.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ MockRequestInfo::MockRequestInfo()
ON_CALL(*this, responseCode()).WillByDefault(ReturnPointee(&response_code_));
ON_CALL(*this, bytesReceived()).WillByDefault(ReturnPointee(&bytes_received_));
ON_CALL(*this, bytesSent()).WillByDefault(ReturnPointee(&bytes_sent_));
ON_CALL(*this, dynamicMetadata()).WillByDefault(ReturnRef(metadata_));
}

MockRequestInfo::~MockRequestInfo() {}
Expand Down
1 change: 1 addition & 0 deletions test/mocks/request_info/mocks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ class MockRequestInfo : public RequestInfo {
absl::optional<uint32_t> response_code_;
uint64_t bytes_received_{};
uint64_t bytes_sent_{};
envoy::api::v2::core::Metadata metadata_;
};

} // namespace RequestInfo
Expand Down
1 change: 1 addition & 0 deletions test/mocks/router/mocks.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,7 @@ class MockMetadataMatchCriteria : public MetadataMatchCriteria {
// Router::MetadataMatchCriteria
MOCK_CONST_METHOD0(metadataMatchCriteria,
const std::vector<MetadataMatchCriterionConstSharedPtr>&());
MOCK_CONST_METHOD1(mergeMatchCriteria, MetadataMatchCriteriaConstPtr(const ProtobufWkt::Struct&));
};

class MockPathMatchCriterion : public PathMatchCriterion {
Expand Down