diff --git a/BUILD b/BUILD
index afac470d5b67fc08f39c28f10263f6daf35bf5db..c7b669997868d836341924535373ce08ceca1c18 100644
--- a/BUILD
+++ b/BUILD
@@ -256,15 +256,24 @@ cc_library(
     "src/core/ext/transport/chttp2/transport/timeout_encoding.h",
     "src/core/ext/transport/chttp2/transport/varint.h",
     "src/core/ext/transport/chttp2/alpn/alpn.h",
-    "src/core/lib/security/auth_filters.h",
-    "src/core/lib/security/b64.h",
-    "src/core/lib/security/credentials.h",
-    "src/core/lib/security/handshake.h",
-    "src/core/lib/security/json_token.h",
-    "src/core/lib/security/jwt_verifier.h",
-    "src/core/lib/security/secure_endpoint.h",
-    "src/core/lib/security/security_connector.h",
-    "src/core/lib/security/security_context.h",
+    "src/core/lib/security/context/security_context.h",
+    "src/core/lib/security/credentials/composite/composite_credentials.h",
+    "src/core/lib/security/credentials/credentials.h",
+    "src/core/lib/security/credentials/fake/fake_credentials.h",
+    "src/core/lib/security/credentials/google_default/google_default_credentials.h",
+    "src/core/lib/security/credentials/iam/iam_credentials.h",
+    "src/core/lib/security/credentials/jwt/json_token.h",
+    "src/core/lib/security/credentials/jwt/jwt_credentials.h",
+    "src/core/lib/security/credentials/jwt/jwt_verifier.h",
+    "src/core/lib/security/credentials/oauth2/oauth2_credentials.h",
+    "src/core/lib/security/credentials/plugin/plugin_credentials.h",
+    "src/core/lib/security/credentials/ssl/ssl_credentials.h",
+    "src/core/lib/security/transport/auth_filters.h",
+    "src/core/lib/security/transport/handshake.h",
+    "src/core/lib/security/transport/secure_endpoint.h",
+    "src/core/lib/security/transport/security_connector.h",
+    "src/core/lib/security/util/b64.h",
+    "src/core/lib/security/util/json_util.h",
     "src/core/lib/tsi/fake_transport_security.h",
     "src/core/lib/tsi/ssl_transport_security.h",
     "src/core/lib/tsi/ssl_types.h",
@@ -294,6 +303,7 @@ cc_library(
     "src/core/ext/census/aggregation.h",
     "src/core/ext/census/census_interface.h",
     "src/core/ext/census/census_rpc_stats.h",
+    "src/core/ext/census/gen/census.pb.h",
     "src/core/ext/census/grpc_filter.h",
     "src/core/ext/census/mlog.h",
     "src/core/ext/census/rpc_metric_id.h",
@@ -404,20 +414,28 @@ cc_library(
     "src/core/ext/transport/chttp2/transport/writing.c",
     "src/core/ext/transport/chttp2/alpn/alpn.c",
     "src/core/lib/http/httpcli_security_connector.c",
-    "src/core/lib/security/b64.c",
-    "src/core/lib/security/client_auth_filter.c",
-    "src/core/lib/security/credentials.c",
-    "src/core/lib/security/credentials_metadata.c",
-    "src/core/lib/security/credentials_posix.c",
-    "src/core/lib/security/credentials_win32.c",
-    "src/core/lib/security/google_default_credentials.c",
-    "src/core/lib/security/handshake.c",
-    "src/core/lib/security/json_token.c",
-    "src/core/lib/security/jwt_verifier.c",
-    "src/core/lib/security/secure_endpoint.c",
-    "src/core/lib/security/security_connector.c",
-    "src/core/lib/security/security_context.c",
-    "src/core/lib/security/server_auth_filter.c",
+    "src/core/lib/security/context/security_context.c",
+    "src/core/lib/security/credentials/composite/composite_credentials.c",
+    "src/core/lib/security/credentials/credentials.c",
+    "src/core/lib/security/credentials/credentials_metadata.c",
+    "src/core/lib/security/credentials/fake/fake_credentials.c",
+    "src/core/lib/security/credentials/google_default/credentials_posix.c",
+    "src/core/lib/security/credentials/google_default/credentials_win32.c",
+    "src/core/lib/security/credentials/google_default/google_default_credentials.c",
+    "src/core/lib/security/credentials/iam/iam_credentials.c",
+    "src/core/lib/security/credentials/jwt/json_token.c",
+    "src/core/lib/security/credentials/jwt/jwt_credentials.c",
+    "src/core/lib/security/credentials/jwt/jwt_verifier.c",
+    "src/core/lib/security/credentials/oauth2/oauth2_credentials.c",
+    "src/core/lib/security/credentials/plugin/plugin_credentials.c",
+    "src/core/lib/security/credentials/ssl/ssl_credentials.c",
+    "src/core/lib/security/transport/client_auth_filter.c",
+    "src/core/lib/security/transport/handshake.c",
+    "src/core/lib/security/transport/secure_endpoint.c",
+    "src/core/lib/security/transport/security_connector.c",
+    "src/core/lib/security/transport/server_auth_filter.c",
+    "src/core/lib/security/util/b64.c",
+    "src/core/lib/security/util/json_util.c",
     "src/core/lib/surface/init_secure.c",
     "src/core/lib/tsi/fake_transport_security.c",
     "src/core/lib/tsi/ssl_transport_security.c",
@@ -456,6 +474,7 @@ cc_library(
     "src/core/ext/load_reporting/load_reporting.c",
     "src/core/ext/load_reporting/load_reporting_filter.c",
     "src/core/ext/census/context.c",
+    "src/core/ext/census/gen/census.pb.c",
     "src/core/ext/census/grpc_context.c",
     "src/core/ext/census/grpc_filter.c",
     "src/core/ext/census/grpc_plugin.c",
@@ -636,6 +655,7 @@ cc_library(
     "src/core/ext/census/aggregation.h",
     "src/core/ext/census/census_interface.h",
     "src/core/ext/census/census_rpc_stats.h",
+    "src/core/ext/census/gen/census.pb.h",
     "src/core/ext/census/grpc_filter.h",
     "src/core/ext/census/mlog.h",
     "src/core/ext/census/rpc_metric_id.h",
@@ -775,6 +795,7 @@ cc_library(
     "src/core/ext/lb_policy/pick_first/pick_first.c",
     "src/core/ext/lb_policy/round_robin/round_robin.c",
     "src/core/ext/census/context.c",
+    "src/core/ext/census/gen/census.pb.c",
     "src/core/ext/census/grpc_context.c",
     "src/core/ext/census/grpc_filter.c",
     "src/core/ext/census/grpc_plugin.c",
@@ -1442,20 +1463,28 @@ objc_library(
     "src/core/ext/transport/chttp2/transport/writing.c",
     "src/core/ext/transport/chttp2/alpn/alpn.c",
     "src/core/lib/http/httpcli_security_connector.c",
-    "src/core/lib/security/b64.c",
-    "src/core/lib/security/client_auth_filter.c",
-    "src/core/lib/security/credentials.c",
-    "src/core/lib/security/credentials_metadata.c",
-    "src/core/lib/security/credentials_posix.c",
-    "src/core/lib/security/credentials_win32.c",
-    "src/core/lib/security/google_default_credentials.c",
-    "src/core/lib/security/handshake.c",
-    "src/core/lib/security/json_token.c",
-    "src/core/lib/security/jwt_verifier.c",
-    "src/core/lib/security/secure_endpoint.c",
-    "src/core/lib/security/security_connector.c",
-    "src/core/lib/security/security_context.c",
-    "src/core/lib/security/server_auth_filter.c",
+    "src/core/lib/security/context/security_context.c",
+    "src/core/lib/security/credentials/composite/composite_credentials.c",
+    "src/core/lib/security/credentials/credentials.c",
+    "src/core/lib/security/credentials/credentials_metadata.c",
+    "src/core/lib/security/credentials/fake/fake_credentials.c",
+    "src/core/lib/security/credentials/google_default/credentials_posix.c",
+    "src/core/lib/security/credentials/google_default/credentials_win32.c",
+    "src/core/lib/security/credentials/google_default/google_default_credentials.c",
+    "src/core/lib/security/credentials/iam/iam_credentials.c",
+    "src/core/lib/security/credentials/jwt/json_token.c",
+    "src/core/lib/security/credentials/jwt/jwt_credentials.c",
+    "src/core/lib/security/credentials/jwt/jwt_verifier.c",
+    "src/core/lib/security/credentials/oauth2/oauth2_credentials.c",
+    "src/core/lib/security/credentials/plugin/plugin_credentials.c",
+    "src/core/lib/security/credentials/ssl/ssl_credentials.c",
+    "src/core/lib/security/transport/client_auth_filter.c",
+    "src/core/lib/security/transport/handshake.c",
+    "src/core/lib/security/transport/secure_endpoint.c",
+    "src/core/lib/security/transport/security_connector.c",
+    "src/core/lib/security/transport/server_auth_filter.c",
+    "src/core/lib/security/util/b64.c",
+    "src/core/lib/security/util/json_util.c",
     "src/core/lib/surface/init_secure.c",
     "src/core/lib/tsi/fake_transport_security.c",
     "src/core/lib/tsi/ssl_transport_security.c",
@@ -1494,6 +1523,7 @@ objc_library(
     "src/core/ext/load_reporting/load_reporting.c",
     "src/core/ext/load_reporting/load_reporting_filter.c",
     "src/core/ext/census/context.c",
+    "src/core/ext/census/gen/census.pb.c",
     "src/core/ext/census/grpc_context.c",
     "src/core/ext/census/grpc_filter.c",
     "src/core/ext/census/grpc_plugin.c",
@@ -1630,15 +1660,24 @@ objc_library(
     "src/core/ext/transport/chttp2/transport/timeout_encoding.h",
     "src/core/ext/transport/chttp2/transport/varint.h",
     "src/core/ext/transport/chttp2/alpn/alpn.h",
-    "src/core/lib/security/auth_filters.h",
-    "src/core/lib/security/b64.h",
-    "src/core/lib/security/credentials.h",
-    "src/core/lib/security/handshake.h",
-    "src/core/lib/security/json_token.h",
-    "src/core/lib/security/jwt_verifier.h",
-    "src/core/lib/security/secure_endpoint.h",
-    "src/core/lib/security/security_connector.h",
-    "src/core/lib/security/security_context.h",
+    "src/core/lib/security/context/security_context.h",
+    "src/core/lib/security/credentials/composite/composite_credentials.h",
+    "src/core/lib/security/credentials/credentials.h",
+    "src/core/lib/security/credentials/fake/fake_credentials.h",
+    "src/core/lib/security/credentials/google_default/google_default_credentials.h",
+    "src/core/lib/security/credentials/iam/iam_credentials.h",
+    "src/core/lib/security/credentials/jwt/json_token.h",
+    "src/core/lib/security/credentials/jwt/jwt_credentials.h",
+    "src/core/lib/security/credentials/jwt/jwt_verifier.h",
+    "src/core/lib/security/credentials/oauth2/oauth2_credentials.h",
+    "src/core/lib/security/credentials/plugin/plugin_credentials.h",
+    "src/core/lib/security/credentials/ssl/ssl_credentials.h",
+    "src/core/lib/security/transport/auth_filters.h",
+    "src/core/lib/security/transport/handshake.h",
+    "src/core/lib/security/transport/secure_endpoint.h",
+    "src/core/lib/security/transport/security_connector.h",
+    "src/core/lib/security/util/b64.h",
+    "src/core/lib/security/util/json_util.h",
     "src/core/lib/tsi/fake_transport_security.h",
     "src/core/lib/tsi/ssl_transport_security.h",
     "src/core/lib/tsi/ssl_types.h",
@@ -1668,6 +1707,7 @@ objc_library(
     "src/core/ext/census/aggregation.h",
     "src/core/ext/census/census_interface.h",
     "src/core/ext/census/census_rpc_stats.h",
+    "src/core/ext/census/gen/census.pb.h",
     "src/core/ext/census/grpc_filter.h",
     "src/core/ext/census/mlog.h",
     "src/core/ext/census/rpc_metric_id.h",
diff --git a/Makefile b/Makefile
index d046fec21e82e0e4b449dc0d22a607fb80728f76..971c86de88db1e2703be5c701c5d0e6b974ffa85 100644
--- a/Makefile
+++ b/Makefile
@@ -215,9 +215,9 @@ CC_mutrace = $(DEFAULT_CC)
 CXX_mutrace = $(DEFAULT_CXX)
 LD_mutrace = $(DEFAULT_CC)
 LDXX_mutrace = $(DEFAULT_CXX)
-CPPFLAGS_mutrace = -O0
+CPPFLAGS_mutrace = -O3 -fno-omit-frame-pointer
 LDFLAGS_mutrace = -rdynamic
-DEFINES_mutrace = _DEBUG DEBUG
+DEFINES_mutrace = NDEBUG
 
 VALID_CONFIG_memcheck = 1
 CC_memcheck = $(DEFAULT_CC)
@@ -1000,8 +1000,6 @@ uri_parser_test: $(BINDIR)/$(CONFIG)/uri_parser_test
 workqueue_test: $(BINDIR)/$(CONFIG)/workqueue_test
 alarm_cpp_test: $(BINDIR)/$(CONFIG)/alarm_cpp_test
 async_end2end_test: $(BINDIR)/$(CONFIG)/async_end2end_test
-async_streaming_ping_pong_test: $(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test
-async_unary_ping_pong_test: $(BINDIR)/$(CONFIG)/async_unary_ping_pong_test
 auth_property_iterator_test: $(BINDIR)/$(CONFIG)/auth_property_iterator_test
 channel_arguments_test: $(BINDIR)/$(CONFIG)/channel_arguments_test
 cli_call_test: $(BINDIR)/$(CONFIG)/cli_call_test
@@ -1015,7 +1013,6 @@ cxx_slice_test: $(BINDIR)/$(CONFIG)/cxx_slice_test
 cxx_string_ref_test: $(BINDIR)/$(CONFIG)/cxx_string_ref_test
 cxx_time_test: $(BINDIR)/$(CONFIG)/cxx_time_test
 end2end_test: $(BINDIR)/$(CONFIG)/end2end_test
-generic_async_streaming_ping_pong_test: $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test
 generic_end2end_test: $(BINDIR)/$(CONFIG)/generic_end2end_test
 golden_file_test: $(BINDIR)/$(CONFIG)/golden_file_test
 grpc_cli: $(BINDIR)/$(CONFIG)/grpc_cli
@@ -1036,7 +1033,6 @@ mock_test: $(BINDIR)/$(CONFIG)/mock_test
 qps_interarrival_test: $(BINDIR)/$(CONFIG)/qps_interarrival_test
 qps_json_driver: $(BINDIR)/$(CONFIG)/qps_json_driver
 qps_openloop_test: $(BINDIR)/$(CONFIG)/qps_openloop_test
-qps_test: $(BINDIR)/$(CONFIG)/qps_test
 qps_worker: $(BINDIR)/$(CONFIG)/qps_worker
 reconnect_interop_client: $(BINDIR)/$(CONFIG)/reconnect_interop_client
 reconnect_interop_server: $(BINDIR)/$(CONFIG)/reconnect_interop_server
@@ -1049,8 +1045,6 @@ shutdown_test: $(BINDIR)/$(CONFIG)/shutdown_test
 status_test: $(BINDIR)/$(CONFIG)/status_test
 streaming_throughput_test: $(BINDIR)/$(CONFIG)/streaming_throughput_test
 stress_test: $(BINDIR)/$(CONFIG)/stress_test
-sync_streaming_ping_pong_test: $(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test
-sync_unary_ping_pong_test: $(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test
 thread_stress_test: $(BINDIR)/$(CONFIG)/thread_stress_test
 zookeeper_test: $(BINDIR)/$(CONFIG)/zookeeper_test
 public_headers_must_be_c89: $(BINDIR)/$(CONFIG)/public_headers_must_be_c89
@@ -1379,8 +1373,6 @@ buildtests_c: privatelibs_c \
 buildtests_cxx: buildtests_zookeeper privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/alarm_cpp_test \
   $(BINDIR)/$(CONFIG)/async_end2end_test \
-  $(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test \
-  $(BINDIR)/$(CONFIG)/async_unary_ping_pong_test \
   $(BINDIR)/$(CONFIG)/auth_property_iterator_test \
   $(BINDIR)/$(CONFIG)/channel_arguments_test \
   $(BINDIR)/$(CONFIG)/cli_call_test \
@@ -1394,7 +1386,6 @@ buildtests_cxx: buildtests_zookeeper privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/cxx_string_ref_test \
   $(BINDIR)/$(CONFIG)/cxx_time_test \
   $(BINDIR)/$(CONFIG)/end2end_test \
-  $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test \
   $(BINDIR)/$(CONFIG)/generic_end2end_test \
   $(BINDIR)/$(CONFIG)/golden_file_test \
   $(BINDIR)/$(CONFIG)/grpc_cli \
@@ -1409,7 +1400,6 @@ buildtests_cxx: buildtests_zookeeper privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/qps_interarrival_test \
   $(BINDIR)/$(CONFIG)/qps_json_driver \
   $(BINDIR)/$(CONFIG)/qps_openloop_test \
-  $(BINDIR)/$(CONFIG)/qps_test \
   $(BINDIR)/$(CONFIG)/qps_worker \
   $(BINDIR)/$(CONFIG)/reconnect_interop_client \
   $(BINDIR)/$(CONFIG)/reconnect_interop_server \
@@ -1422,8 +1412,6 @@ buildtests_cxx: buildtests_zookeeper privatelibs_cxx \
   $(BINDIR)/$(CONFIG)/status_test \
   $(BINDIR)/$(CONFIG)/streaming_throughput_test \
   $(BINDIR)/$(CONFIG)/stress_test \
-  $(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test \
-  $(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test \
   $(BINDIR)/$(CONFIG)/thread_stress_test \
   $(BINDIR)/$(CONFIG)/boringssl_aes_test \
   $(BINDIR)/$(CONFIG)/boringssl_asn1_test \
@@ -1699,10 +1687,6 @@ test_cxx: test_zookeeper buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/alarm_cpp_test || ( echo test alarm_cpp_test failed ; exit 1 )
 	$(E) "[RUN]     Testing async_end2end_test"
 	$(Q) $(BINDIR)/$(CONFIG)/async_end2end_test || ( echo test async_end2end_test failed ; exit 1 )
-	$(E) "[RUN]     Testing async_streaming_ping_pong_test"
-	$(Q) $(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test || ( echo test async_streaming_ping_pong_test failed ; exit 1 )
-	$(E) "[RUN]     Testing async_unary_ping_pong_test"
-	$(Q) $(BINDIR)/$(CONFIG)/async_unary_ping_pong_test || ( echo test async_unary_ping_pong_test failed ; exit 1 )
 	$(E) "[RUN]     Testing auth_property_iterator_test"
 	$(Q) $(BINDIR)/$(CONFIG)/auth_property_iterator_test || ( echo test auth_property_iterator_test failed ; exit 1 )
 	$(E) "[RUN]     Testing channel_arguments_test"
@@ -1727,8 +1711,6 @@ test_cxx: test_zookeeper buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/cxx_time_test || ( echo test cxx_time_test failed ; exit 1 )
 	$(E) "[RUN]     Testing end2end_test"
 	$(Q) $(BINDIR)/$(CONFIG)/end2end_test || ( echo test end2end_test failed ; exit 1 )
-	$(E) "[RUN]     Testing generic_async_streaming_ping_pong_test"
-	$(Q) $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test || ( echo test generic_async_streaming_ping_pong_test failed ; exit 1 )
 	$(E) "[RUN]     Testing generic_end2end_test"
 	$(Q) $(BINDIR)/$(CONFIG)/generic_end2end_test || ( echo test generic_end2end_test failed ; exit 1 )
 	$(E) "[RUN]     Testing golden_file_test"
@@ -1743,8 +1725,6 @@ test_cxx: test_zookeeper buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/mock_test || ( echo test mock_test failed ; exit 1 )
 	$(E) "[RUN]     Testing qps_openloop_test"
 	$(Q) $(BINDIR)/$(CONFIG)/qps_openloop_test || ( echo test qps_openloop_test failed ; exit 1 )
-	$(E) "[RUN]     Testing qps_test"
-	$(Q) $(BINDIR)/$(CONFIG)/qps_test || ( echo test qps_test failed ; exit 1 )
 	$(E) "[RUN]     Testing secure_auth_context_test"
 	$(Q) $(BINDIR)/$(CONFIG)/secure_auth_context_test || ( echo test secure_auth_context_test failed ; exit 1 )
 	$(E) "[RUN]     Testing secure_sync_unary_ping_pong_test"
@@ -1759,10 +1739,6 @@ test_cxx: test_zookeeper buildtests_cxx
 	$(Q) $(BINDIR)/$(CONFIG)/status_test || ( echo test status_test failed ; exit 1 )
 	$(E) "[RUN]     Testing streaming_throughput_test"
 	$(Q) $(BINDIR)/$(CONFIG)/streaming_throughput_test || ( echo test streaming_throughput_test failed ; exit 1 )
-	$(E) "[RUN]     Testing sync_streaming_ping_pong_test"
-	$(Q) $(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test || ( echo test sync_streaming_ping_pong_test failed ; exit 1 )
-	$(E) "[RUN]     Testing sync_unary_ping_pong_test"
-	$(Q) $(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test || ( echo test sync_unary_ping_pong_test failed ; exit 1 )
 	$(E) "[RUN]     Testing thread_stress_test"
 	$(Q) $(BINDIR)/$(CONFIG)/thread_stress_test || ( echo test thread_stress_test failed ; exit 1 )
 
@@ -1876,7 +1852,7 @@ $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_unsecure.pc:
 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_zookeeper.pc:
 	$(E) "[MAKE]    Generating $@"
 	$(Q) mkdir -p $(@D)
-	$(Q) echo -e "$(GRPC_ZOOKEEPER_PC_FILE)" >$@
+	$(Q) echo "$(GRPC_ZOOKEEPER_PC_FILE)" | tr , '\n' >$@
 
 $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc:
 	$(E) "[MAKE]    Generating $@"
@@ -2318,17 +2294,19 @@ ifeq ($(INSTALL_OK),true)
 	@echo "Your system looks ready to go."
 	@echo
 else
-	@echo "We couldn't find protoc 3.0.0+ installed on your system. While this"
-	@echo "won't prevent grpc from working, you won't be able to compile"
-	@echo "and run any meaningful code with it."
+	@echo "Warning: it looks like protoc 3.0.0+ isn't installed on your system,"
+	@echo "which means that you won't be able to compile .proto files for use"
+	@echo "with gRPC."
 	@echo
+	@echo "If you are just using pre-compiled protocol buffers, or you otherwise"
+	@echo "have no need to compile .proto files, you can ignore this."
 	@echo
-	@echo "Please download and install protobuf 3.0.0+ from:"
+	@echo "If you do need protobuf for some reason, you can download and install"
+	@echo "it from:"
 	@echo
 	@echo "   https://github.com/google/protobuf/releases"
 	@echo
-	@echo "Once you've done so, or if you think this message is in error,"
-	@echo "you can re-run this check by doing:"
+	@echo "Once you've done so, you can re-run this check by doing:"
 	@echo
 	@echo "   make verify-install"
 endif
@@ -2603,20 +2581,28 @@ LIBGRPC_SRC = \
     src/core/ext/transport/chttp2/transport/writing.c \
     src/core/ext/transport/chttp2/alpn/alpn.c \
     src/core/lib/http/httpcli_security_connector.c \
-    src/core/lib/security/b64.c \
-    src/core/lib/security/client_auth_filter.c \
-    src/core/lib/security/credentials.c \
-    src/core/lib/security/credentials_metadata.c \
-    src/core/lib/security/credentials_posix.c \
-    src/core/lib/security/credentials_win32.c \
-    src/core/lib/security/google_default_credentials.c \
-    src/core/lib/security/handshake.c \
-    src/core/lib/security/json_token.c \
-    src/core/lib/security/jwt_verifier.c \
-    src/core/lib/security/secure_endpoint.c \
-    src/core/lib/security/security_connector.c \
-    src/core/lib/security/security_context.c \
-    src/core/lib/security/server_auth_filter.c \
+    src/core/lib/security/context/security_context.c \
+    src/core/lib/security/credentials/composite/composite_credentials.c \
+    src/core/lib/security/credentials/credentials.c \
+    src/core/lib/security/credentials/credentials_metadata.c \
+    src/core/lib/security/credentials/fake/fake_credentials.c \
+    src/core/lib/security/credentials/google_default/credentials_posix.c \
+    src/core/lib/security/credentials/google_default/credentials_win32.c \
+    src/core/lib/security/credentials/google_default/google_default_credentials.c \
+    src/core/lib/security/credentials/iam/iam_credentials.c \
+    src/core/lib/security/credentials/jwt/json_token.c \
+    src/core/lib/security/credentials/jwt/jwt_credentials.c \
+    src/core/lib/security/credentials/jwt/jwt_verifier.c \
+    src/core/lib/security/credentials/oauth2/oauth2_credentials.c \
+    src/core/lib/security/credentials/plugin/plugin_credentials.c \
+    src/core/lib/security/credentials/ssl/ssl_credentials.c \
+    src/core/lib/security/transport/client_auth_filter.c \
+    src/core/lib/security/transport/handshake.c \
+    src/core/lib/security/transport/secure_endpoint.c \
+    src/core/lib/security/transport/security_connector.c \
+    src/core/lib/security/transport/server_auth_filter.c \
+    src/core/lib/security/util/b64.c \
+    src/core/lib/security/util/json_util.c \
     src/core/lib/surface/init_secure.c \
     src/core/lib/tsi/fake_transport_security.c \
     src/core/lib/tsi/ssl_transport_security.c \
@@ -2658,6 +2644,7 @@ LIBGRPC_SRC = \
     src/core/ext/load_reporting/load_reporting.c \
     src/core/ext/load_reporting/load_reporting_filter.c \
     src/core/ext/census/context.c \
+    src/core/ext/census/gen/census.pb.c \
     src/core/ext/census/grpc_context.c \
     src/core/ext/census/grpc_filter.c \
     src/core/ext/census/grpc_plugin.c \
@@ -2983,6 +2970,7 @@ LIBGRPC_UNSECURE_SRC = \
     src/core/ext/lb_policy/pick_first/pick_first.c \
     src/core/ext/lb_policy/round_robin/round_robin.c \
     src/core/ext/census/context.c \
+    src/core/ext/census/gen/census.pb.c \
     src/core/ext/census/grpc_context.c \
     src/core/ext/census/grpc_filter.c \
     src/core/ext/census/grpc_plugin.c \
@@ -9725,92 +9713,6 @@ endif
 endif
 
 
-ASYNC_STREAMING_PING_PONG_TEST_SRC = \
-    test/cpp/qps/async_streaming_ping_pong_test.cc \
-
-ASYNC_STREAMING_PING_PONG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ASYNC_STREAMING_PING_PONG_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test: $(PROTOBUF_DEP) $(ASYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(ASYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/async_streaming_ping_pong_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/async_streaming_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_async_streaming_ping_pong_test: $(ASYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(ASYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
-ASYNC_UNARY_PING_PONG_TEST_SRC = \
-    test/cpp/qps/async_unary_ping_pong_test.cc \
-
-ASYNC_UNARY_PING_PONG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(ASYNC_UNARY_PING_PONG_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/async_unary_ping_pong_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/async_unary_ping_pong_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/async_unary_ping_pong_test: $(PROTOBUF_DEP) $(ASYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(ASYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/async_unary_ping_pong_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/async_unary_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_async_unary_ping_pong_test: $(ASYNC_UNARY_PING_PONG_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(ASYNC_UNARY_PING_PONG_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 AUTH_PROPERTY_ITERATOR_TEST_SRC = \
     test/cpp/common/auth_property_iterator_test.cc \
 
@@ -10416,49 +10318,6 @@ endif
 endif
 
 
-GENERIC_ASYNC_STREAMING_PING_PONG_TEST_SRC = \
-    test/cpp/qps/generic_async_streaming_ping_pong_test.cc \
-
-GENERIC_ASYNC_STREAMING_PING_PONG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(GENERIC_ASYNC_STREAMING_PING_PONG_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test: $(PROTOBUF_DEP) $(GENERIC_ASYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(GENERIC_ASYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/generic_async_streaming_ping_pong_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/generic_async_streaming_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_generic_async_streaming_ping_pong_test: $(GENERIC_ASYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(GENERIC_ASYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 GENERIC_END2END_TEST_SRC = \
     test/cpp/end2end/generic_end2end_test.cc \
 
@@ -11238,49 +11097,6 @@ endif
 endif
 
 
-QPS_TEST_SRC = \
-    test/cpp/qps/qps_test.cc \
-
-QPS_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(QPS_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/qps_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/qps_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/qps_test: $(PROTOBUF_DEP) $(QPS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(QPS_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/qps_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/qps_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_config.a
-
-deps_qps_test: $(QPS_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(QPS_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 QPS_WORKER_SRC = \
     test/cpp/qps/worker.cc \
 
@@ -11842,92 +11658,6 @@ $(OBJDIR)/$(CONFIG)/test/cpp/interop/stress_test.o: $(GENDIR)/src/proto/grpc/tes
 $(OBJDIR)/$(CONFIG)/test/cpp/util/metrics_server.o: $(GENDIR)/src/proto/grpc/testing/empty.pb.cc $(GENDIR)/src/proto/grpc/testing/empty.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.pb.cc $(GENDIR)/src/proto/grpc/testing/messages.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.pb.cc $(GENDIR)/src/proto/grpc/testing/metrics.grpc.pb.cc $(GENDIR)/src/proto/grpc/testing/test.pb.cc $(GENDIR)/src/proto/grpc/testing/test.grpc.pb.cc
 
 
-SYNC_STREAMING_PING_PONG_TEST_SRC = \
-    test/cpp/qps/sync_streaming_ping_pong_test.cc \
-
-SYNC_STREAMING_PING_PONG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SYNC_STREAMING_PING_PONG_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test: $(PROTOBUF_DEP) $(SYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(SYNC_STREAMING_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/sync_streaming_ping_pong_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/sync_streaming_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_sync_streaming_ping_pong_test: $(SYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(SYNC_STREAMING_PING_PONG_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
-SYNC_UNARY_PING_PONG_TEST_SRC = \
-    test/cpp/qps/sync_unary_ping_pong_test.cc \
-
-SYNC_UNARY_PING_PONG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(SYNC_UNARY_PING_PONG_TEST_SRC))))
-ifeq ($(NO_SECURE),true)
-
-# You can't build secure targets if you don't have OpenSSL.
-
-$(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test: openssl_dep_error
-
-else
-
-
-
-
-ifeq ($(NO_PROTOBUF),true)
-
-# You can't build the protoc plugins or protobuf-enabled targets if you don't have protobuf 3.0.0+.
-
-$(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test: protobuf_dep_error
-
-else
-
-$(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test: $(PROTOBUF_DEP) $(SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-	$(E) "[LD]      Linking $@"
-	$(Q) mkdir -p `dirname $@`
-	$(Q) $(LDXX) $(LDFLAGS) $(SYNC_UNARY_PING_PONG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBSXX) $(LDLIBS_PROTOBUF) $(LDLIBS) $(LDLIBS_SECURE) $(GTEST_LIB) -o $(BINDIR)/$(CONFIG)/sync_unary_ping_pong_test
-
-endif
-
-endif
-
-$(OBJDIR)/$(CONFIG)/test/cpp/qps/sync_unary_ping_pong_test.o:  $(LIBDIR)/$(CONFIG)/libqps.a $(LIBDIR)/$(CONFIG)/libgrpc++_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc++.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a
-
-deps_sync_unary_ping_pong_test: $(SYNC_UNARY_PING_PONG_TEST_OBJS:.o=.dep)
-
-ifneq ($(NO_SECURE),true)
-ifneq ($(NO_DEPS),true)
--include $(SYNC_UNARY_PING_PONG_TEST_OBJS:.o=.dep)
-endif
-endif
-
-
 THREAD_STRESS_TEST_SRC = \
     test/cpp/end2end/thread_stress_test.cc \
 
@@ -14469,20 +14199,28 @@ src/core/ext/transport/cronet/client/secure/cronet_channel_create.c: $(OPENSSL_D
 src/core/ext/transport/cronet/transport/cronet_api_dummy.c: $(OPENSSL_DEP)
 src/core/ext/transport/cronet/transport/cronet_transport.c: $(OPENSSL_DEP)
 src/core/lib/http/httpcli_security_connector.c: $(OPENSSL_DEP)
-src/core/lib/security/b64.c: $(OPENSSL_DEP)
-src/core/lib/security/client_auth_filter.c: $(OPENSSL_DEP)
-src/core/lib/security/credentials.c: $(OPENSSL_DEP)
-src/core/lib/security/credentials_metadata.c: $(OPENSSL_DEP)
-src/core/lib/security/credentials_posix.c: $(OPENSSL_DEP)
-src/core/lib/security/credentials_win32.c: $(OPENSSL_DEP)
-src/core/lib/security/google_default_credentials.c: $(OPENSSL_DEP)
-src/core/lib/security/handshake.c: $(OPENSSL_DEP)
-src/core/lib/security/json_token.c: $(OPENSSL_DEP)
-src/core/lib/security/jwt_verifier.c: $(OPENSSL_DEP)
-src/core/lib/security/secure_endpoint.c: $(OPENSSL_DEP)
-src/core/lib/security/security_connector.c: $(OPENSSL_DEP)
-src/core/lib/security/security_context.c: $(OPENSSL_DEP)
-src/core/lib/security/server_auth_filter.c: $(OPENSSL_DEP)
+src/core/lib/security/context/security_context.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/composite/composite_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/credentials_metadata.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/fake/fake_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/google_default/credentials_posix.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/google_default/credentials_win32.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/google_default/google_default_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/iam/iam_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/jwt/json_token.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/jwt/jwt_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/jwt/jwt_verifier.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/oauth2/oauth2_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/plugin/plugin_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/credentials/ssl/ssl_credentials.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/client_auth_filter.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/handshake.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/secure_endpoint.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/security_connector.c: $(OPENSSL_DEP)
+src/core/lib/security/transport/server_auth_filter.c: $(OPENSSL_DEP)
+src/core/lib/security/util/b64.c: $(OPENSSL_DEP)
+src/core/lib/security/util/json_util.c: $(OPENSSL_DEP)
 src/core/lib/surface/init_secure.c: $(OPENSSL_DEP)
 src/core/lib/tsi/fake_transport_security.c: $(OPENSSL_DEP)
 src/core/lib/tsi/ssl_transport_security.c: $(OPENSSL_DEP)
diff --git a/README.md b/README.md
index 3ee2b9f5caab187f62271517fd9d350a42b7bc16..3283517df1700a84d4b16838a44b7c24b2debd35 100644
--- a/README.md
+++ b/README.md
@@ -11,10 +11,12 @@ Copyright 2015 Google Inc.
 
 You can find more detailed documentation and examples in the [doc](doc) and [examples](examples) directories respectively.
 
-#Installation
+#Installation & Testing
 
 See [INSTALL](INSTALL.md) for installation instructions for various platforms.
 
+See [tools/run_tests](tools/run_tests) for more guidance on how to run various test suites (e.g. unit tests, interop tests, benchmarks)
+
 #Repository Structure & Status
 
 This repository contains source code for gRPC libraries for multiple languages written on top of shared C core library [src/core] (src/core).
diff --git a/binding.gyp b/binding.gyp
index 86d24bdf3810fe70f52159272c421912ff9f69aa..4da426964196dfd1b451be03855d7c88edd7d1e5 100644
--- a/binding.gyp
+++ b/binding.gyp
@@ -670,20 +670,28 @@
         'src/core/ext/transport/chttp2/transport/writing.c',
         'src/core/ext/transport/chttp2/alpn/alpn.c',
         'src/core/lib/http/httpcli_security_connector.c',
-        'src/core/lib/security/b64.c',
-        'src/core/lib/security/client_auth_filter.c',
-        'src/core/lib/security/credentials.c',
-        'src/core/lib/security/credentials_metadata.c',
-        'src/core/lib/security/credentials_posix.c',
-        'src/core/lib/security/credentials_win32.c',
-        'src/core/lib/security/google_default_credentials.c',
-        'src/core/lib/security/handshake.c',
-        'src/core/lib/security/json_token.c',
-        'src/core/lib/security/jwt_verifier.c',
-        'src/core/lib/security/secure_endpoint.c',
-        'src/core/lib/security/security_connector.c',
-        'src/core/lib/security/security_context.c',
-        'src/core/lib/security/server_auth_filter.c',
+        'src/core/lib/security/context/security_context.c',
+        'src/core/lib/security/credentials/composite/composite_credentials.c',
+        'src/core/lib/security/credentials/credentials.c',
+        'src/core/lib/security/credentials/credentials_metadata.c',
+        'src/core/lib/security/credentials/fake/fake_credentials.c',
+        'src/core/lib/security/credentials/google_default/credentials_posix.c',
+        'src/core/lib/security/credentials/google_default/credentials_win32.c',
+        'src/core/lib/security/credentials/google_default/google_default_credentials.c',
+        'src/core/lib/security/credentials/iam/iam_credentials.c',
+        'src/core/lib/security/credentials/jwt/json_token.c',
+        'src/core/lib/security/credentials/jwt/jwt_credentials.c',
+        'src/core/lib/security/credentials/jwt/jwt_verifier.c',
+        'src/core/lib/security/credentials/oauth2/oauth2_credentials.c',
+        'src/core/lib/security/credentials/plugin/plugin_credentials.c',
+        'src/core/lib/security/credentials/ssl/ssl_credentials.c',
+        'src/core/lib/security/transport/client_auth_filter.c',
+        'src/core/lib/security/transport/handshake.c',
+        'src/core/lib/security/transport/secure_endpoint.c',
+        'src/core/lib/security/transport/security_connector.c',
+        'src/core/lib/security/transport/server_auth_filter.c',
+        'src/core/lib/security/util/b64.c',
+        'src/core/lib/security/util/json_util.c',
         'src/core/lib/surface/init_secure.c',
         'src/core/lib/tsi/fake_transport_security.c',
         'src/core/lib/tsi/ssl_transport_security.c',
@@ -725,6 +733,7 @@
         'src/core/ext/load_reporting/load_reporting.c',
         'src/core/ext/load_reporting/load_reporting_filter.c',
         'src/core/ext/census/context.c',
+        'src/core/ext/census/gen/census.pb.c',
         'src/core/ext/census/grpc_context.c',
         'src/core/ext/census/grpc_filter.c',
         'src/core/ext/census/grpc_plugin.c',
diff --git a/build.yaml b/build.yaml
index 9685d46fa915978df16d0ad1075d953516fcc1c7..14f087e2e9ca28ebe6820a40061b67f31f771b8a 100644
--- a/build.yaml
+++ b/build.yaml
@@ -16,11 +16,13 @@ filegroups:
   - src/core/ext/census/aggregation.h
   - src/core/ext/census/census_interface.h
   - src/core/ext/census/census_rpc_stats.h
+  - src/core/ext/census/gen/census.pb.h
   - src/core/ext/census/grpc_filter.h
   - src/core/ext/census/mlog.h
   - src/core/ext/census/rpc_metric_id.h
   src:
   - src/core/ext/census/context.c
+  - src/core/ext/census/gen/census.pb.c
   - src/core/ext/census/grpc_context.c
   - src/core/ext/census/grpc_filter.c
   - src/core/ext/census/grpc_plugin.c
@@ -32,6 +34,7 @@ filegroups:
   plugin: census_grpc_plugin
   uses:
   - grpc_base
+  - nanopb
 - name: gpr_base
   public_headers:
   - include/grpc/support/alloc.h
@@ -416,31 +419,48 @@ filegroups:
   - include/grpc/grpc_security.h
   - include/grpc/grpc_security_constants.h
   headers:
-  - src/core/lib/security/auth_filters.h
-  - src/core/lib/security/b64.h
-  - src/core/lib/security/credentials.h
-  - src/core/lib/security/handshake.h
-  - src/core/lib/security/json_token.h
-  - src/core/lib/security/jwt_verifier.h
-  - src/core/lib/security/secure_endpoint.h
-  - src/core/lib/security/security_connector.h
-  - src/core/lib/security/security_context.h
+  - src/core/lib/security/context/security_context.h
+  - src/core/lib/security/credentials/composite/composite_credentials.h
+  - src/core/lib/security/credentials/credentials.h
+  - src/core/lib/security/credentials/fake/fake_credentials.h
+  - src/core/lib/security/credentials/google_default/google_default_credentials.h
+  - src/core/lib/security/credentials/iam/iam_credentials.h
+  - src/core/lib/security/credentials/jwt/json_token.h
+  - src/core/lib/security/credentials/jwt/jwt_credentials.h
+  - src/core/lib/security/credentials/jwt/jwt_verifier.h
+  - src/core/lib/security/credentials/oauth2/oauth2_credentials.h
+  - src/core/lib/security/credentials/plugin/plugin_credentials.h
+  - src/core/lib/security/credentials/ssl/ssl_credentials.h
+  - src/core/lib/security/transport/auth_filters.h
+  - src/core/lib/security/transport/handshake.h
+  - src/core/lib/security/transport/secure_endpoint.h
+  - src/core/lib/security/transport/security_connector.h
+  - src/core/lib/security/util/b64.h
+  - src/core/lib/security/util/json_util.h
   src:
   - src/core/lib/http/httpcli_security_connector.c
-  - src/core/lib/security/b64.c
-  - src/core/lib/security/client_auth_filter.c
-  - src/core/lib/security/credentials.c
-  - src/core/lib/security/credentials_metadata.c
-  - src/core/lib/security/credentials_posix.c
-  - src/core/lib/security/credentials_win32.c
-  - src/core/lib/security/google_default_credentials.c
-  - src/core/lib/security/handshake.c
-  - src/core/lib/security/json_token.c
-  - src/core/lib/security/jwt_verifier.c
-  - src/core/lib/security/secure_endpoint.c
-  - src/core/lib/security/security_connector.c
-  - src/core/lib/security/security_context.c
-  - src/core/lib/security/server_auth_filter.c
+  - src/core/lib/security/context/security_context.c
+  - src/core/lib/security/credentials/composite/composite_credentials.c
+  - src/core/lib/security/credentials/credentials.c
+  - src/core/lib/security/credentials/credentials_metadata.c
+  - src/core/lib/security/credentials/fake/fake_credentials.c
+  - src/core/lib/security/credentials/google_default/credentials_posix.c
+  - src/core/lib/security/credentials/google_default/credentials_win32.c
+  - src/core/lib/security/credentials/google_default/google_default_credentials.c
+  - src/core/lib/security/credentials/iam/iam_credentials.c
+  - src/core/lib/security/credentials/jwt/json_token.c
+  - src/core/lib/security/credentials/jwt/jwt_credentials.c
+  - src/core/lib/security/credentials/jwt/jwt_verifier.c
+  - src/core/lib/security/credentials/oauth2/oauth2_credentials.c
+  - src/core/lib/security/credentials/plugin/plugin_credentials.c
+  - src/core/lib/security/credentials/ssl/ssl_credentials.c
+  - src/core/lib/security/transport/client_auth_filter.c
+  - src/core/lib/security/transport/handshake.c
+  - src/core/lib/security/transport/secure_endpoint.c
+  - src/core/lib/security/transport/security_connector.c
+  - src/core/lib/security/transport/server_auth_filter.c
+  - src/core/lib/security/util/b64.c
+  - src/core/lib/security/util/json_util.c
   - src/core/lib/surface/init_secure.c
   secure: true
   uses:
@@ -2351,40 +2371,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-- name: async_streaming_ping_pong_test
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/async_streaming_ping_pong_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
-- name: async_unary_ping_pong_test
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/async_unary_ping_pong_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
 - name: auth_property_iterator_test
   gtest: true
   build: test
@@ -2552,23 +2538,6 @@ targets:
   - grpc
   - gpr_test_util
   - gpr
-- name: generic_async_streaming_ping_pong_test
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/generic_async_streaming_ping_pong_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
 - name: generic_end2end_test
   gtest: true
   build: test
@@ -2840,25 +2809,6 @@ targets:
   - mac
   - linux
   - posix
-- name: qps_test
-  cpu_cost: 10
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/qps_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  - grpc++_test_config
-  platforms:
-  - mac
-  - linux
-  - posix
 - name: qps_worker
   build: test
   run: false
@@ -3054,40 +3004,6 @@ targets:
   - gpr_test_util
   - gpr
   - grpc++_test_config
-- name: sync_streaming_ping_pong_test
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/sync_streaming_ping_pong_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
-- name: sync_unary_ping_pong_test
-  build: test
-  language: c++
-  src:
-  - test/cpp/qps/sync_unary_ping_pong_test.cc
-  deps:
-  - qps
-  - grpc++_test_util
-  - grpc_test_util
-  - grpc++
-  - grpc
-  - gpr_test_util
-  - gpr
-  platforms:
-  - mac
-  - linux
-  - posix
 - name: thread_stress_test
   gtest: true
   cpu_cost: 100
@@ -3258,8 +3174,8 @@ configs:
     compile_the_world: true
     timeout_multiplier: 4
   mutrace:
-    CPPFLAGS: -O0
-    DEFINES: _DEBUG DEBUG
+    CPPFLAGS: -O3 -fno-omit-frame-pointer
+    DEFINES: NDEBUG
     LDFLAGS: -rdynamic
   opt:
     CPPFLAGS: -O2
@@ -3290,7 +3206,7 @@ configs:
     LDXX: clang++
     compile_the_world: true
     test_environ:
-      UBSAN_OPTIONS: print_stacktrace=1
+      UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1
     timeout_multiplier: 1.5
 defaults:
   boringssl:
diff --git a/config.m4 b/config.m4
index 53ec206b999a130e74b07cf85f1054c1018f8f8b..3ac78ca95692e040e71e55b4d0cfdbfa98f2b02e 100644
--- a/config.m4
+++ b/config.m4
@@ -189,20 +189,28 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/transport/chttp2/transport/writing.c \
     src/core/ext/transport/chttp2/alpn/alpn.c \
     src/core/lib/http/httpcli_security_connector.c \
-    src/core/lib/security/b64.c \
-    src/core/lib/security/client_auth_filter.c \
-    src/core/lib/security/credentials.c \
-    src/core/lib/security/credentials_metadata.c \
-    src/core/lib/security/credentials_posix.c \
-    src/core/lib/security/credentials_win32.c \
-    src/core/lib/security/google_default_credentials.c \
-    src/core/lib/security/handshake.c \
-    src/core/lib/security/json_token.c \
-    src/core/lib/security/jwt_verifier.c \
-    src/core/lib/security/secure_endpoint.c \
-    src/core/lib/security/security_connector.c \
-    src/core/lib/security/security_context.c \
-    src/core/lib/security/server_auth_filter.c \
+    src/core/lib/security/context/security_context.c \
+    src/core/lib/security/credentials/composite/composite_credentials.c \
+    src/core/lib/security/credentials/credentials.c \
+    src/core/lib/security/credentials/credentials_metadata.c \
+    src/core/lib/security/credentials/fake/fake_credentials.c \
+    src/core/lib/security/credentials/google_default/credentials_posix.c \
+    src/core/lib/security/credentials/google_default/credentials_win32.c \
+    src/core/lib/security/credentials/google_default/google_default_credentials.c \
+    src/core/lib/security/credentials/iam/iam_credentials.c \
+    src/core/lib/security/credentials/jwt/json_token.c \
+    src/core/lib/security/credentials/jwt/jwt_credentials.c \
+    src/core/lib/security/credentials/jwt/jwt_verifier.c \
+    src/core/lib/security/credentials/oauth2/oauth2_credentials.c \
+    src/core/lib/security/credentials/plugin/plugin_credentials.c \
+    src/core/lib/security/credentials/ssl/ssl_credentials.c \
+    src/core/lib/security/transport/client_auth_filter.c \
+    src/core/lib/security/transport/handshake.c \
+    src/core/lib/security/transport/secure_endpoint.c \
+    src/core/lib/security/transport/security_connector.c \
+    src/core/lib/security/transport/server_auth_filter.c \
+    src/core/lib/security/util/b64.c \
+    src/core/lib/security/util/json_util.c \
     src/core/lib/surface/init_secure.c \
     src/core/lib/tsi/fake_transport_security.c \
     src/core/lib/tsi/ssl_transport_security.c \
@@ -244,6 +252,7 @@ if test "$PHP_GRPC" != "no"; then
     src/core/ext/load_reporting/load_reporting.c \
     src/core/ext/load_reporting/load_reporting_filter.c \
     src/core/ext/census/context.c \
+    src/core/ext/census/gen/census.pb.c \
     src/core/ext/census/grpc_context.c \
     src/core/ext/census/grpc_filter.c \
     src/core/ext/census/grpc_plugin.c \
@@ -559,6 +568,7 @@ if test "$PHP_GRPC" != "no"; then
 
   PHP_ADD_BUILD_DIR($ext_builddir/src/boringssl)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/census)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/census/gen)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/client_config)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/grpclb)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1)
@@ -582,7 +592,18 @@ if test "$PHP_GRPC" != "no"; then
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/iomgr)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/json)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/profiling)
-  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/context)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/composite)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/fake)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/google_default)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/iam)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/jwt)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/oauth2)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/plugin)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/credentials/ssl)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/transport)
+  PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/security/util)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/support)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/surface)
   PHP_ADD_BUILD_DIR($ext_builddir/src/core/lib/transport)
diff --git a/doc/connectivity-semantics-and-api.md b/doc/connectivity-semantics-and-api.md
index 5427900394b451a9891251525af9c2be90d2bed8..cc007eaae3c8f0148e4260f7236f0d1a83325872 100644
--- a/doc/connectivity-semantics-and-api.md
+++ b/doc/connectivity-semantics-and-api.md
@@ -101,7 +101,7 @@ corresponding reasons. Empty cells denote disallowed transitions.
     <td>Shutdown triggered by application.</td>
   </tr>
   <tr>
-    <th>FATAL_FAILURE</th>
+    <th>SHUTDOWN</th>
     <td></td>
     <td></td>
     <td></td>
diff --git a/examples/cpp/helloworld/Makefile b/examples/cpp/helloworld/Makefile
index 58a82dbb8312611ea278d85a2e9bdb991994b5a8..780e5e427a7677db21aacfc3f6976ca363e12f1d 100644
--- a/examples/cpp/helloworld/Makefile
+++ b/examples/cpp/helloworld/Makefile
@@ -32,7 +32,7 @@
 CXX = g++
 CPPFLAGS += -I/usr/local/include -pthread
 CXXFLAGS += -std=c++11
-LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++` -lprotobuf -lpthread -ldl
+LDFLAGS += -L/usr/local/lib `pkg-config --libs grpc++ grpc` -lprotobuf -lpthread -ldl
 PROTOC = protoc
 GRPC_CPP_PLUGIN = grpc_cpp_plugin
 GRPC_CPP_PLUGIN_PATH ?= `which $(GRPC_CPP_PLUGIN)`
diff --git a/gRPC.podspec b/gRPC.podspec
index 3c349d7ae497a0af91d26d31777cd283e3ad0725..c90a947ce3d554cca75f079a9393ec07dbb3bff4 100644
--- a/gRPC.podspec
+++ b/gRPC.podspec
@@ -259,15 +259,24 @@ Pod::Spec.new do |s|
                       'src/core/ext/transport/chttp2/transport/timeout_encoding.h',
                       'src/core/ext/transport/chttp2/transport/varint.h',
                       'src/core/ext/transport/chttp2/alpn/alpn.h',
-                      'src/core/lib/security/auth_filters.h',
-                      'src/core/lib/security/b64.h',
-                      'src/core/lib/security/credentials.h',
-                      'src/core/lib/security/handshake.h',
-                      'src/core/lib/security/json_token.h',
-                      'src/core/lib/security/jwt_verifier.h',
-                      'src/core/lib/security/secure_endpoint.h',
-                      'src/core/lib/security/security_connector.h',
-                      'src/core/lib/security/security_context.h',
+                      'src/core/lib/security/context/security_context.h',
+                      'src/core/lib/security/credentials/composite/composite_credentials.h',
+                      'src/core/lib/security/credentials/credentials.h',
+                      'src/core/lib/security/credentials/fake/fake_credentials.h',
+                      'src/core/lib/security/credentials/google_default/google_default_credentials.h',
+                      'src/core/lib/security/credentials/iam/iam_credentials.h',
+                      'src/core/lib/security/credentials/jwt/json_token.h',
+                      'src/core/lib/security/credentials/jwt/jwt_credentials.h',
+                      'src/core/lib/security/credentials/jwt/jwt_verifier.h',
+                      'src/core/lib/security/credentials/oauth2/oauth2_credentials.h',
+                      'src/core/lib/security/credentials/plugin/plugin_credentials.h',
+                      'src/core/lib/security/credentials/ssl/ssl_credentials.h',
+                      'src/core/lib/security/transport/auth_filters.h',
+                      'src/core/lib/security/transport/handshake.h',
+                      'src/core/lib/security/transport/secure_endpoint.h',
+                      'src/core/lib/security/transport/security_connector.h',
+                      'src/core/lib/security/util/b64.h',
+                      'src/core/lib/security/util/json_util.h',
                       'src/core/lib/tsi/fake_transport_security.h',
                       'src/core/lib/tsi/ssl_transport_security.h',
                       'src/core/lib/tsi/ssl_types.h',
@@ -301,6 +310,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/census/aggregation.h',
                       'src/core/ext/census/census_interface.h',
                       'src/core/ext/census/census_rpc_stats.h',
+                      'src/core/ext/census/gen/census.pb.h',
                       'src/core/ext/census/grpc_filter.h',
                       'src/core/ext/census/mlog.h',
                       'src/core/ext/census/rpc_metric_id.h',
@@ -441,20 +451,28 @@ Pod::Spec.new do |s|
                       'src/core/ext/transport/chttp2/transport/writing.c',
                       'src/core/ext/transport/chttp2/alpn/alpn.c',
                       'src/core/lib/http/httpcli_security_connector.c',
-                      'src/core/lib/security/b64.c',
-                      'src/core/lib/security/client_auth_filter.c',
-                      'src/core/lib/security/credentials.c',
-                      'src/core/lib/security/credentials_metadata.c',
-                      'src/core/lib/security/credentials_posix.c',
-                      'src/core/lib/security/credentials_win32.c',
-                      'src/core/lib/security/google_default_credentials.c',
-                      'src/core/lib/security/handshake.c',
-                      'src/core/lib/security/json_token.c',
-                      'src/core/lib/security/jwt_verifier.c',
-                      'src/core/lib/security/secure_endpoint.c',
-                      'src/core/lib/security/security_connector.c',
-                      'src/core/lib/security/security_context.c',
-                      'src/core/lib/security/server_auth_filter.c',
+                      'src/core/lib/security/context/security_context.c',
+                      'src/core/lib/security/credentials/composite/composite_credentials.c',
+                      'src/core/lib/security/credentials/credentials.c',
+                      'src/core/lib/security/credentials/credentials_metadata.c',
+                      'src/core/lib/security/credentials/fake/fake_credentials.c',
+                      'src/core/lib/security/credentials/google_default/credentials_posix.c',
+                      'src/core/lib/security/credentials/google_default/credentials_win32.c',
+                      'src/core/lib/security/credentials/google_default/google_default_credentials.c',
+                      'src/core/lib/security/credentials/iam/iam_credentials.c',
+                      'src/core/lib/security/credentials/jwt/json_token.c',
+                      'src/core/lib/security/credentials/jwt/jwt_credentials.c',
+                      'src/core/lib/security/credentials/jwt/jwt_verifier.c',
+                      'src/core/lib/security/credentials/oauth2/oauth2_credentials.c',
+                      'src/core/lib/security/credentials/plugin/plugin_credentials.c',
+                      'src/core/lib/security/credentials/ssl/ssl_credentials.c',
+                      'src/core/lib/security/transport/client_auth_filter.c',
+                      'src/core/lib/security/transport/handshake.c',
+                      'src/core/lib/security/transport/secure_endpoint.c',
+                      'src/core/lib/security/transport/security_connector.c',
+                      'src/core/lib/security/transport/server_auth_filter.c',
+                      'src/core/lib/security/util/b64.c',
+                      'src/core/lib/security/util/json_util.c',
                       'src/core/lib/surface/init_secure.c',
                       'src/core/lib/tsi/fake_transport_security.c',
                       'src/core/lib/tsi/ssl_transport_security.c',
@@ -496,6 +514,7 @@ Pod::Spec.new do |s|
                       'src/core/ext/load_reporting/load_reporting.c',
                       'src/core/ext/load_reporting/load_reporting_filter.c',
                       'src/core/ext/census/context.c',
+                      'src/core/ext/census/gen/census.pb.c',
                       'src/core/ext/census/grpc_context.c',
                       'src/core/ext/census/grpc_filter.c',
                       'src/core/ext/census/grpc_plugin.c',
@@ -613,15 +632,24 @@ Pod::Spec.new do |s|
                               'src/core/ext/transport/chttp2/transport/timeout_encoding.h',
                               'src/core/ext/transport/chttp2/transport/varint.h',
                               'src/core/ext/transport/chttp2/alpn/alpn.h',
-                              'src/core/lib/security/auth_filters.h',
-                              'src/core/lib/security/b64.h',
-                              'src/core/lib/security/credentials.h',
-                              'src/core/lib/security/handshake.h',
-                              'src/core/lib/security/json_token.h',
-                              'src/core/lib/security/jwt_verifier.h',
-                              'src/core/lib/security/secure_endpoint.h',
-                              'src/core/lib/security/security_connector.h',
-                              'src/core/lib/security/security_context.h',
+                              'src/core/lib/security/context/security_context.h',
+                              'src/core/lib/security/credentials/composite/composite_credentials.h',
+                              'src/core/lib/security/credentials/credentials.h',
+                              'src/core/lib/security/credentials/fake/fake_credentials.h',
+                              'src/core/lib/security/credentials/google_default/google_default_credentials.h',
+                              'src/core/lib/security/credentials/iam/iam_credentials.h',
+                              'src/core/lib/security/credentials/jwt/json_token.h',
+                              'src/core/lib/security/credentials/jwt/jwt_credentials.h',
+                              'src/core/lib/security/credentials/jwt/jwt_verifier.h',
+                              'src/core/lib/security/credentials/oauth2/oauth2_credentials.h',
+                              'src/core/lib/security/credentials/plugin/plugin_credentials.h',
+                              'src/core/lib/security/credentials/ssl/ssl_credentials.h',
+                              'src/core/lib/security/transport/auth_filters.h',
+                              'src/core/lib/security/transport/handshake.h',
+                              'src/core/lib/security/transport/secure_endpoint.h',
+                              'src/core/lib/security/transport/security_connector.h',
+                              'src/core/lib/security/util/b64.h',
+                              'src/core/lib/security/util/json_util.h',
                               'src/core/lib/tsi/fake_transport_security.h',
                               'src/core/lib/tsi/ssl_transport_security.h',
                               'src/core/lib/tsi/ssl_types.h',
@@ -655,6 +683,7 @@ Pod::Spec.new do |s|
                               'src/core/ext/census/aggregation.h',
                               'src/core/ext/census/census_interface.h',
                               'src/core/ext/census/census_rpc_stats.h',
+                              'src/core/ext/census/gen/census.pb.h',
                               'src/core/ext/census/grpc_filter.h',
                               'src/core/ext/census/mlog.h',
                               'src/core/ext/census/rpc_metric_id.h'
diff --git a/grpc.def b/grpc.def
index 09a94a6cd0c502fe2b053e40f38908504d35ab15..b811f0ff1296da2e5d393d677573e95a37b62da7 100644
--- a/grpc.def
+++ b/grpc.def
@@ -77,6 +77,7 @@ EXPORTS
     grpc_server_request_registered_call
     grpc_server_create
     grpc_server_register_completion_queue
+    grpc_server_register_non_listening_completion_queue
     grpc_server_add_insecure_http2_port
     grpc_server_start
     grpc_server_shutdown_and_notify
diff --git a/grpc.gemspec b/grpc.gemspec
index 03f1a194c2eea4d23540271c4366b19209ddf388..ede3bbf23b29eec197e1682ab102cc6c255d6bb0 100755
--- a/grpc.gemspec
+++ b/grpc.gemspec
@@ -268,15 +268,24 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/transport/chttp2/transport/timeout_encoding.h )
   s.files += %w( src/core/ext/transport/chttp2/transport/varint.h )
   s.files += %w( src/core/ext/transport/chttp2/alpn/alpn.h )
-  s.files += %w( src/core/lib/security/auth_filters.h )
-  s.files += %w( src/core/lib/security/b64.h )
-  s.files += %w( src/core/lib/security/credentials.h )
-  s.files += %w( src/core/lib/security/handshake.h )
-  s.files += %w( src/core/lib/security/json_token.h )
-  s.files += %w( src/core/lib/security/jwt_verifier.h )
-  s.files += %w( src/core/lib/security/secure_endpoint.h )
-  s.files += %w( src/core/lib/security/security_connector.h )
-  s.files += %w( src/core/lib/security/security_context.h )
+  s.files += %w( src/core/lib/security/context/security_context.h )
+  s.files += %w( src/core/lib/security/credentials/composite/composite_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/credentials.h )
+  s.files += %w( src/core/lib/security/credentials/fake/fake_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/google_default/google_default_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/iam/iam_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/jwt/json_token.h )
+  s.files += %w( src/core/lib/security/credentials/jwt/jwt_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/jwt/jwt_verifier.h )
+  s.files += %w( src/core/lib/security/credentials/oauth2/oauth2_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.h )
+  s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.h )
+  s.files += %w( src/core/lib/security/transport/auth_filters.h )
+  s.files += %w( src/core/lib/security/transport/handshake.h )
+  s.files += %w( src/core/lib/security/transport/secure_endpoint.h )
+  s.files += %w( src/core/lib/security/transport/security_connector.h )
+  s.files += %w( src/core/lib/security/util/b64.h )
+  s.files += %w( src/core/lib/security/util/json_util.h )
   s.files += %w( src/core/lib/tsi/fake_transport_security.h )
   s.files += %w( src/core/lib/tsi/ssl_transport_security.h )
   s.files += %w( src/core/lib/tsi/ssl_types.h )
@@ -310,6 +319,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/census/aggregation.h )
   s.files += %w( src/core/ext/census/census_interface.h )
   s.files += %w( src/core/ext/census/census_rpc_stats.h )
+  s.files += %w( src/core/ext/census/gen/census.pb.h )
   s.files += %w( src/core/ext/census/grpc_filter.h )
   s.files += %w( src/core/ext/census/mlog.h )
   s.files += %w( src/core/ext/census/rpc_metric_id.h )
@@ -420,20 +430,28 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/transport/chttp2/transport/writing.c )
   s.files += %w( src/core/ext/transport/chttp2/alpn/alpn.c )
   s.files += %w( src/core/lib/http/httpcli_security_connector.c )
-  s.files += %w( src/core/lib/security/b64.c )
-  s.files += %w( src/core/lib/security/client_auth_filter.c )
-  s.files += %w( src/core/lib/security/credentials.c )
-  s.files += %w( src/core/lib/security/credentials_metadata.c )
-  s.files += %w( src/core/lib/security/credentials_posix.c )
-  s.files += %w( src/core/lib/security/credentials_win32.c )
-  s.files += %w( src/core/lib/security/google_default_credentials.c )
-  s.files += %w( src/core/lib/security/handshake.c )
-  s.files += %w( src/core/lib/security/json_token.c )
-  s.files += %w( src/core/lib/security/jwt_verifier.c )
-  s.files += %w( src/core/lib/security/secure_endpoint.c )
-  s.files += %w( src/core/lib/security/security_connector.c )
-  s.files += %w( src/core/lib/security/security_context.c )
-  s.files += %w( src/core/lib/security/server_auth_filter.c )
+  s.files += %w( src/core/lib/security/context/security_context.c )
+  s.files += %w( src/core/lib/security/credentials/composite/composite_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/credentials.c )
+  s.files += %w( src/core/lib/security/credentials/credentials_metadata.c )
+  s.files += %w( src/core/lib/security/credentials/fake/fake_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/google_default/credentials_posix.c )
+  s.files += %w( src/core/lib/security/credentials/google_default/credentials_win32.c )
+  s.files += %w( src/core/lib/security/credentials/google_default/google_default_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/iam/iam_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/jwt/json_token.c )
+  s.files += %w( src/core/lib/security/credentials/jwt/jwt_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/jwt/jwt_verifier.c )
+  s.files += %w( src/core/lib/security/credentials/oauth2/oauth2_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/plugin/plugin_credentials.c )
+  s.files += %w( src/core/lib/security/credentials/ssl/ssl_credentials.c )
+  s.files += %w( src/core/lib/security/transport/client_auth_filter.c )
+  s.files += %w( src/core/lib/security/transport/handshake.c )
+  s.files += %w( src/core/lib/security/transport/secure_endpoint.c )
+  s.files += %w( src/core/lib/security/transport/security_connector.c )
+  s.files += %w( src/core/lib/security/transport/server_auth_filter.c )
+  s.files += %w( src/core/lib/security/util/b64.c )
+  s.files += %w( src/core/lib/security/util/json_util.c )
   s.files += %w( src/core/lib/surface/init_secure.c )
   s.files += %w( src/core/lib/tsi/fake_transport_security.c )
   s.files += %w( src/core/lib/tsi/ssl_transport_security.c )
@@ -475,6 +493,7 @@ Gem::Specification.new do |s|
   s.files += %w( src/core/ext/load_reporting/load_reporting.c )
   s.files += %w( src/core/ext/load_reporting/load_reporting_filter.c )
   s.files += %w( src/core/ext/census/context.c )
+  s.files += %w( src/core/ext/census/gen/census.pb.c )
   s.files += %w( src/core/ext/census/grpc_context.c )
   s.files += %w( src/core/ext/census/grpc_filter.c )
   s.files += %w( src/core/ext/census/grpc_plugin.c )
diff --git a/include/grpc++/impl/codegen/completion_queue.h b/include/grpc++/impl/codegen/completion_queue.h
index 56864d6d53633d2761b128daee13c19d9891f3aa..1b84b4470500728abfeaffbc443259ac83e42799 100644
--- a/include/grpc++/impl/codegen/completion_queue.h
+++ b/include/grpc++/impl/codegen/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -222,9 +222,18 @@ class CompletionQueue : private GrpcLibraryCodegen {
 /// A specific type of completion queue used by the processing of notifications
 /// by servers. Instantiated by \a ServerBuilder.
 class ServerCompletionQueue : public CompletionQueue {
+ public:
+  bool IsFrequentlyPolled() { return is_frequently_polled_; }
+
  private:
+  bool is_frequently_polled_;
   friend class ServerBuilder;
-  ServerCompletionQueue() {}
+  /// \param is_frequently_polled Informs the GPRC library about whether the
+  /// server completion queue would be actively polled (by calling Next() or
+  /// AsyncNext()). By default all server completion queues are assumed to be
+  /// frequently polled.
+  ServerCompletionQueue(bool is_frequently_polled = true)
+      : is_frequently_polled_(is_frequently_polled) {}
 };
 
 }  // namespace grpc
diff --git a/include/grpc++/server_builder.h b/include/grpc++/server_builder.h
index ad629521cbe415e5a8e7b0f0f24d8f6aa72f2158..8525cb70cbb8a5cf43fd427e89323623ea6f525c 100644
--- a/include/grpc++/server_builder.h
+++ b/include/grpc++/server_builder.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -108,7 +108,15 @@ class ServerBuilder {
   /// Add a completion queue for handling asynchronous services
   /// Caller is required to keep this completion queue live until
   /// the server is destroyed.
-  std::unique_ptr<ServerCompletionQueue> AddCompletionQueue();
+  ///
+  /// \param is_frequently_polled This is an optional parameter to inform GRPC
+  /// library about whether this completion queue would be frequently polled
+  /// (i.e by calling Next() or AsyncNext()). The default value is 'true' and is
+  /// the recommended setting. Setting this to 'false' (i.e not polling the
+  /// completion queue frequently) will have a significantly negative
+  /// performance impact and hence should not be used in production use cases.
+  std::unique_ptr<ServerCompletionQueue> AddCompletionQueue(
+      bool is_frequently_polled = true);
 
   /// Return a running server which is ready for processing calls.
   std::unique_ptr<Server> BuildAndStart();
diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h
index 0ca28c0fef1824980707ba64afac4b68ac50465b..6f7a67b715ea3c29686f6574b6df33d2229d3339 100644
--- a/include/grpc/grpc.h
+++ b/include/grpc/grpc.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -334,6 +334,15 @@ GRPCAPI void grpc_server_register_completion_queue(grpc_server *server,
                                                    grpc_completion_queue *cq,
                                                    void *reserved);
 
+/** Register a non-listening completion queue with the server. This API is
+    similar to grpc_server_register_completion_queue except that the server will
+    not use this completion_queue to listen to any incoming channels.
+
+    Registering a non-listening completion queue will have negative performance
+    impact and hence this API is not recommended for production use cases. */
+GRPCAPI void grpc_server_register_non_listening_completion_queue(
+    grpc_server *server, grpc_completion_queue *q, void *reserved);
+
 /** Add a HTTP2 over plaintext over tcp listener.
     Returns bound port number on success, 0 on failure.
     REQUIRES: server not started */
diff --git a/package.xml b/package.xml
index 2d2b97426693b20e5da66efec78e16b38a204079..d90a8f53dc71c89bb818414199bb37021a01be87 100644
--- a/package.xml
+++ b/package.xml
@@ -275,15 +275,24 @@
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/timeout_encoding.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/varint.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/alpn/alpn.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/auth_filters.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/b64.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/credentials.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/handshake.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/json_token.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/jwt_verifier.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/secure_endpoint.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/security_connector.h" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/security_context.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/context/security_context.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/composite/composite_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/fake/fake_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/google_default/google_default_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/iam/iam_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/json_token.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/jwt_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/jwt_verifier.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/oauth2/oauth2_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/plugin/plugin_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/ssl/ssl_credentials.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/auth_filters.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/handshake.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/secure_endpoint.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/security_connector.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/util/b64.h" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/util/json_util.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/fake_transport_security.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/ssl_transport_security.h" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/ssl_types.h" role="src" />
@@ -317,6 +326,7 @@
     <file baseinstalldir="/" name="src/core/ext/census/aggregation.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/census_interface.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/census_rpc_stats.h" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/census/gen/census.pb.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/grpc_filter.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/mlog.h" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/rpc_metric_id.h" role="src" />
@@ -427,20 +437,28 @@
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/transport/writing.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/transport/chttp2/alpn/alpn.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/http/httpcli_security_connector.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/b64.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/client_auth_filter.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/credentials.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/credentials_metadata.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/credentials_posix.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/credentials_win32.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/google_default_credentials.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/handshake.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/json_token.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/jwt_verifier.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/secure_endpoint.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/security_connector.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/security_context.c" role="src" />
-    <file baseinstalldir="/" name="src/core/lib/security/server_auth_filter.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/context/security_context.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/composite/composite_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/credentials_metadata.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/fake/fake_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/google_default/credentials_posix.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/google_default/credentials_win32.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/google_default/google_default_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/iam/iam_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/json_token.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/jwt_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/jwt/jwt_verifier.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/oauth2/oauth2_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/plugin/plugin_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/credentials/ssl/ssl_credentials.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/client_auth_filter.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/handshake.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/secure_endpoint.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/security_connector.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/transport/server_auth_filter.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/util/b64.c" role="src" />
+    <file baseinstalldir="/" name="src/core/lib/security/util/json_util.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/surface/init_secure.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/fake_transport_security.c" role="src" />
     <file baseinstalldir="/" name="src/core/lib/tsi/ssl_transport_security.c" role="src" />
@@ -482,6 +500,7 @@
     <file baseinstalldir="/" name="src/core/ext/load_reporting/load_reporting.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/load_reporting/load_reporting_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/context.c" role="src" />
+    <file baseinstalldir="/" name="src/core/ext/census/gen/census.pb.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/grpc_context.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/grpc_filter.c" role="src" />
     <file baseinstalldir="/" name="src/core/ext/census/grpc_plugin.c" role="src" />
diff --git a/src/core/ext/census/gen/README.md b/src/core/ext/census/gen/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..72bef6542d02e6e12d3980ad7323db303ed09cc7
--- /dev/null
+++ b/src/core/ext/census/gen/README.md
@@ -0,0 +1,6 @@
+Files generated for use by Census stats and trace recording subsystem.
+
+#Files
+* census.pb.{h,c} - Generated from src/core/ext/census/census.proto, using the
+  script `tools/codegen/core/gen_nano_proto.sh src/proto/census/census.proto
+  $PWD/src/core/ext/census/gen src/core/ext/census/gen`
diff --git a/src/core/ext/census/gen/census.pb.c b/src/core/ext/census/gen/census.pb.c
new file mode 100644
index 0000000000000000000000000000000000000000..d614636c908706c9d42ed3b04ead0570aa43fd5d
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.c
@@ -0,0 +1,179 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb constant definitions */
+/* Generated by nanopb-0.3.5-dev */
+
+#include "src/core/ext/census/gen/census.pb.h"
+
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+
+
+const pb_field_t google_census_Duration_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Duration, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Duration, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Timestamp_fields[3] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Timestamp, seconds, seconds, 0),
+    PB_FIELD(  2, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Timestamp, nanos, seconds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_fields[5] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Metric, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Metric, description, name, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Metric, unit, description, &google_census_Metric_MeasurementUnit_fields),
+    PB_FIELD(  4, INT32   , OPTIONAL, STATIC  , OTHER, google_census_Metric, id, unit, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_BasicUnit_fields[2] = {
+    PB_FIELD(  1, UENUM   , OPTIONAL, STATIC  , FIRST, google_census_Metric_BasicUnit, type, type, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Metric_MeasurementUnit_fields[4] = {
+    PB_FIELD(  1, INT32   , OPTIONAL, STATIC  , FIRST, google_census_Metric_MeasurementUnit, prefix, prefix, 0),
+    PB_FIELD(  2, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, numerator, prefix, &google_census_Metric_BasicUnit_fields),
+    PB_FIELD(  3, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Metric_MeasurementUnit, denominator, numerator, &google_census_Metric_BasicUnit_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_fields[3] = {
+    PB_ONEOF_FIELD(options,   1, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, bucket_boundaries, bucket_boundaries, &google_census_AggregationDescriptor_BucketBoundaries_fields),
+    PB_ONEOF_FIELD(options,   2, MESSAGE , ONEOF, STATIC  , FIRST, google_census_AggregationDescriptor, interval_boundaries, interval_boundaries, &google_census_AggregationDescriptor_IntervalBoundaries_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_BucketBoundaries, bounds, bounds, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2] = {
+    PB_FIELD(  1, DOUBLE  , REPEATED, CALLBACK, FIRST, google_census_AggregationDescriptor_IntervalBoundaries, window_size, window_size, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_fields[5] = {
+    PB_FIELD(  1, INT64   , OPTIONAL, STATIC  , FIRST, google_census_Distribution, count, count, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution, mean, count, 0),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_Distribution, range, mean, &google_census_Distribution_Range_fields),
+    PB_FIELD(  4, INT64   , REPEATED, CALLBACK, OTHER, google_census_Distribution, bucket_count, range, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Distribution_Range_fields[3] = {
+    PB_FIELD(  1, DOUBLE  , OPTIONAL, STATIC  , FIRST, google_census_Distribution_Range, min, min, 0),
+    PB_FIELD(  2, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_Distribution_Range, max, min, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_fields[2] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_IntervalStats, window, window, &google_census_IntervalStats_Window_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_IntervalStats_Window_fields[4] = {
+    PB_FIELD(  1, MESSAGE , OPTIONAL, STATIC  , FIRST, google_census_IntervalStats_Window, window_size, window_size, &google_census_Duration_fields),
+    PB_FIELD(  2, INT64   , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, count, window_size, 0),
+    PB_FIELD(  3, DOUBLE  , OPTIONAL, STATIC  , OTHER, google_census_IntervalStats_Window, mean, count, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Tag_fields[3] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, STATIC  , FIRST, google_census_Tag, key, key, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, STATIC  , OTHER, google_census_Tag, value, key, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_View_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_View, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_View, description, name, 0),
+    PB_FIELD(  3, INT32   , OPTIONAL, STATIC  , OTHER, google_census_View, metric_id, description, 0),
+    PB_FIELD(  4, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_View, aggregation, metric_id, &google_census_AggregationDescriptor_fields),
+    PB_FIELD(  5, STRING  , REPEATED, CALLBACK, OTHER, google_census_View, tag_key, aggregation, 0),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_Aggregation_fields[6] = {
+    PB_FIELD(  1, STRING  , OPTIONAL, CALLBACK, FIRST, google_census_Aggregation, name, name, 0),
+    PB_FIELD(  2, STRING  , OPTIONAL, CALLBACK, OTHER, google_census_Aggregation, description, name, 0),
+    PB_ONEOF_FIELD(data,   3, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, distribution, description, &google_census_Distribution_fields),
+    PB_ONEOF_FIELD(data,   4, MESSAGE , ONEOF, STATIC  , OTHER, google_census_Aggregation, interval_stats, description, &google_census_IntervalStats_fields),
+    PB_FIELD(  5, MESSAGE , REPEATED, CALLBACK, OTHER, google_census_Aggregation, tag, data.interval_stats, &google_census_Tag_fields),
+    PB_LAST_FIELD
+};
+
+const pb_field_t google_census_ViewAggregations_fields[4] = {
+    PB_FIELD(  1, MESSAGE , REPEATED, CALLBACK, FIRST, google_census_ViewAggregations, aggregation, aggregation, &google_census_Aggregation_fields),
+    PB_FIELD(  2, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, start, aggregation, &google_census_Timestamp_fields),
+    PB_FIELD(  3, MESSAGE , OPTIONAL, STATIC  , OTHER, google_census_ViewAggregations, end, start, &google_census_Timestamp_fields),
+    PB_LAST_FIELD
+};
+
+
+/* Check that field information fits in pb_field_t */
+#if !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_32BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in 8 or 16 bit
+ * field descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Metric, unit) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 65536 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 65536 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 65536 && pb_membersize(google_census_Distribution, range) < 65536 && pb_membersize(google_census_IntervalStats, window) < 65536 && pb_membersize(google_census_IntervalStats_Window, window_size) < 65536 && pb_membersize(google_census_View, aggregation) < 65536 && pb_membersize(google_census_Aggregation, data.distribution) < 65536 && pb_membersize(google_census_Aggregation, data.interval_stats) < 65536 && pb_membersize(google_census_Aggregation, tag) < 65536 && pb_membersize(google_census_ViewAggregations, aggregation) < 65536 && pb_membersize(google_census_ViewAggregations, start) < 65536 && pb_membersize(google_census_ViewAggregations, end) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)
+/* If you get an error here, it means that you need to define PB_FIELD_16BIT
+ * compile-time option. You can do that in pb.h or on compiler command line.
+ * 
+ * The reason you need to do this is that some of your messages contain tag
+ * numbers or field sizes that are larger than what can fit in the default
+ * 8 bit descriptors.
+ */
+PB_STATIC_ASSERT((pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Metric, unit) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, numerator) < 256 && pb_membersize(google_census_Metric_MeasurementUnit, denominator) < 256 && pb_membersize(google_census_AggregationDescriptor, options.bucket_boundaries) < 256 && pb_membersize(google_census_AggregationDescriptor, options.interval_boundaries) < 256 && pb_membersize(google_census_Distribution, range) < 256 && pb_membersize(google_census_IntervalStats, window) < 256 && pb_membersize(google_census_IntervalStats_Window, window_size) < 256 && pb_membersize(google_census_View, aggregation) < 256 && pb_membersize(google_census_Aggregation, data.distribution) < 256 && pb_membersize(google_census_Aggregation, data.interval_stats) < 256 && pb_membersize(google_census_Aggregation, tag) < 256 && pb_membersize(google_census_ViewAggregations, aggregation) < 256 && pb_membersize(google_census_ViewAggregations, start) < 256 && pb_membersize(google_census_ViewAggregations, end) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_google_census_Duration_google_census_Timestamp_google_census_Metric_google_census_Metric_BasicUnit_google_census_Metric_MeasurementUnit_google_census_AggregationDescriptor_google_census_AggregationDescriptor_BucketBoundaries_google_census_AggregationDescriptor_IntervalBoundaries_google_census_Distribution_google_census_Distribution_Range_google_census_IntervalStats_google_census_IntervalStats_Window_google_census_Tag_google_census_View_google_census_Aggregation_google_census_ViewAggregations)
+#endif
+
+
+/* On some platforms (such as AVR), double is really float.
+ * These are not directly supported by nanopb, but see example_avr_double.
+ * To get rid of this error, remove any double fields from your .proto.
+ */
+PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)
+
diff --git a/src/core/ext/census/gen/census.pb.h b/src/core/ext/census/gen/census.pb.h
new file mode 100644
index 0000000000000000000000000000000000000000..d040fe29e74ada3f9d1015513e6e56cb9c307916
--- /dev/null
+++ b/src/core/ext/census/gen/census.pb.h
@@ -0,0 +1,294 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+/* Automatically generated nanopb header */
+/* Generated by nanopb-0.3.5-dev */
+
+#ifndef GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#define GRPC_CORE_EXT_CENSUS_GEN_CENSUS_PB_H
+#include "third_party/nanopb/pb.h"
+#if PB_PROTO_HEADER_VERSION != 30
+#error Regenerate this file with the current version of nanopb generator.
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Enum definitions */
+typedef enum _google_census_Metric_BasicUnit_Measure {
+    google_census_Metric_BasicUnit_Measure_UNKNOWN = 0,
+    google_census_Metric_BasicUnit_Measure_BITS = 1,
+    google_census_Metric_BasicUnit_Measure_BYTES = 2,
+    google_census_Metric_BasicUnit_Measure_SECS = 3,
+    google_census_Metric_BasicUnit_Measure_CORES = 4,
+    google_census_Metric_BasicUnit_Measure_MAX_UNITS = 5
+} google_census_Metric_BasicUnit_Measure;
+
+/* Struct definitions */
+typedef struct _google_census_AggregationDescriptor_BucketBoundaries {
+    pb_callback_t bounds;
+} google_census_AggregationDescriptor_BucketBoundaries;
+
+typedef struct _google_census_AggregationDescriptor_IntervalBoundaries {
+    pb_callback_t window_size;
+} google_census_AggregationDescriptor_IntervalBoundaries;
+
+typedef struct _google_census_IntervalStats {
+    pb_callback_t window;
+} google_census_IntervalStats;
+
+typedef struct _google_census_AggregationDescriptor {
+    pb_size_t which_options;
+    union {
+        google_census_AggregationDescriptor_BucketBoundaries bucket_boundaries;
+        google_census_AggregationDescriptor_IntervalBoundaries interval_boundaries;
+    } options;
+} google_census_AggregationDescriptor;
+
+typedef struct _google_census_Distribution_Range {
+    bool has_min;
+    double min;
+    bool has_max;
+    double max;
+} google_census_Distribution_Range;
+
+typedef struct _google_census_Duration {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Duration;
+
+typedef struct _google_census_Metric_BasicUnit {
+    bool has_type;
+    google_census_Metric_BasicUnit_Measure type;
+} google_census_Metric_BasicUnit;
+
+typedef struct _google_census_Metric_MeasurementUnit {
+    bool has_prefix;
+    int32_t prefix;
+    pb_callback_t numerator;
+    pb_callback_t denominator;
+} google_census_Metric_MeasurementUnit;
+
+typedef struct _google_census_Tag {
+    bool has_key;
+    char key[255];
+    bool has_value;
+    char value[255];
+} google_census_Tag;
+
+typedef struct _google_census_Timestamp {
+    bool has_seconds;
+    int64_t seconds;
+    bool has_nanos;
+    int32_t nanos;
+} google_census_Timestamp;
+
+typedef struct _google_census_Distribution {
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+    bool has_range;
+    google_census_Distribution_Range range;
+    pb_callback_t bucket_count;
+} google_census_Distribution;
+
+typedef struct _google_census_IntervalStats_Window {
+    bool has_window_size;
+    google_census_Duration window_size;
+    bool has_count;
+    int64_t count;
+    bool has_mean;
+    double mean;
+} google_census_IntervalStats_Window;
+
+typedef struct _google_census_Metric {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_unit;
+    google_census_Metric_MeasurementUnit unit;
+    bool has_id;
+    int32_t id;
+} google_census_Metric;
+
+typedef struct _google_census_View {
+    pb_callback_t name;
+    pb_callback_t description;
+    bool has_metric_id;
+    int32_t metric_id;
+    bool has_aggregation;
+    google_census_AggregationDescriptor aggregation;
+    pb_callback_t tag_key;
+} google_census_View;
+
+typedef struct _google_census_ViewAggregations {
+    pb_callback_t aggregation;
+    bool has_start;
+    google_census_Timestamp start;
+    bool has_end;
+    google_census_Timestamp end;
+} google_census_ViewAggregations;
+
+typedef struct _google_census_Aggregation {
+    pb_callback_t name;
+    pb_callback_t description;
+    pb_size_t which_data;
+    union {
+        google_census_Distribution distribution;
+        google_census_IntervalStats interval_stats;
+    } data;
+    pb_callback_t tag;
+} google_census_Aggregation;
+
+/* Default values for struct fields */
+
+/* Initializer values for message structs */
+#define google_census_Duration_init_default      {false, 0, false, 0}
+#define google_census_Timestamp_init_default     {false, 0, false, 0}
+#define google_census_Metric_init_default        {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_default, false, 0}
+#define google_census_Metric_BasicUnit_init_default {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_default {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_default {0, {google_census_AggregationDescriptor_BucketBoundaries_init_default}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_default {{{NULL}, NULL}}
+#define google_census_Distribution_init_default  {false, 0, false, 0, false, google_census_Distribution_Range_init_default, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_default {false, 0, false, 0}
+#define google_census_IntervalStats_init_default {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_default {false, google_census_Duration_init_default, false, 0, false, 0}
+#define google_census_Tag_init_default           {false, "", false, ""}
+#define google_census_View_init_default          {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_default, {{NULL}, NULL}}
+#define google_census_Aggregation_init_default   {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_default}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_default {{{NULL}, NULL}, false, google_census_Timestamp_init_default, false, google_census_Timestamp_init_default}
+#define google_census_Duration_init_zero         {false, 0, false, 0}
+#define google_census_Timestamp_init_zero        {false, 0, false, 0}
+#define google_census_Metric_init_zero           {{{NULL}, NULL}, {{NULL}, NULL}, false, google_census_Metric_MeasurementUnit_init_zero, false, 0}
+#define google_census_Metric_BasicUnit_init_zero {false, (google_census_Metric_BasicUnit_Measure)0}
+#define google_census_Metric_MeasurementUnit_init_zero {false, 0, {{NULL}, NULL}, {{NULL}, NULL}}
+#define google_census_AggregationDescriptor_init_zero {0, {google_census_AggregationDescriptor_BucketBoundaries_init_zero}}
+#define google_census_AggregationDescriptor_BucketBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_AggregationDescriptor_IntervalBoundaries_init_zero {{{NULL}, NULL}}
+#define google_census_Distribution_init_zero     {false, 0, false, 0, false, google_census_Distribution_Range_init_zero, {{NULL}, NULL}}
+#define google_census_Distribution_Range_init_zero {false, 0, false, 0}
+#define google_census_IntervalStats_init_zero    {{{NULL}, NULL}}
+#define google_census_IntervalStats_Window_init_zero {false, google_census_Duration_init_zero, false, 0, false, 0}
+#define google_census_Tag_init_zero              {false, "", false, ""}
+#define google_census_View_init_zero             {{{NULL}, NULL}, {{NULL}, NULL}, false, 0, false, google_census_AggregationDescriptor_init_zero, {{NULL}, NULL}}
+#define google_census_Aggregation_init_zero      {{{NULL}, NULL}, {{NULL}, NULL}, 0, {google_census_Distribution_init_zero}, {{NULL}, NULL}}
+#define google_census_ViewAggregations_init_zero {{{NULL}, NULL}, false, google_census_Timestamp_init_zero, false, google_census_Timestamp_init_zero}
+
+/* Field tags (for use in manual encoding/decoding) */
+#define google_census_AggregationDescriptor_BucketBoundaries_bounds_tag 1
+#define google_census_AggregationDescriptor_IntervalBoundaries_window_size_tag 1
+#define google_census_IntervalStats_window_tag   1
+#define google_census_AggregationDescriptor_bucket_boundaries_tag 1
+
+#define google_census_AggregationDescriptor_interval_boundaries_tag 2
+#define google_census_Distribution_Range_min_tag 1
+#define google_census_Distribution_Range_max_tag 2
+#define google_census_Duration_seconds_tag       1
+#define google_census_Duration_nanos_tag         2
+#define google_census_Metric_BasicUnit_type_tag  1
+#define google_census_Metric_MeasurementUnit_prefix_tag 1
+#define google_census_Metric_MeasurementUnit_numerator_tag 2
+#define google_census_Metric_MeasurementUnit_denominator_tag 3
+#define google_census_Tag_key_tag                1
+#define google_census_Tag_value_tag              2
+#define google_census_Timestamp_seconds_tag      1
+#define google_census_Timestamp_nanos_tag        2
+#define google_census_Distribution_count_tag     1
+#define google_census_Distribution_mean_tag      2
+#define google_census_Distribution_range_tag     3
+#define google_census_Distribution_bucket_count_tag 4
+#define google_census_IntervalStats_Window_window_size_tag 1
+#define google_census_IntervalStats_Window_count_tag 2
+#define google_census_IntervalStats_Window_mean_tag 3
+#define google_census_Metric_name_tag            1
+#define google_census_Metric_description_tag     2
+#define google_census_Metric_unit_tag            3
+#define google_census_Metric_id_tag              4
+#define google_census_View_name_tag              1
+#define google_census_View_description_tag       2
+#define google_census_View_metric_id_tag         3
+#define google_census_View_aggregation_tag       4
+#define google_census_View_tag_key_tag           5
+#define google_census_ViewAggregations_aggregation_tag 1
+#define google_census_ViewAggregations_start_tag 2
+#define google_census_ViewAggregations_end_tag   3
+#define google_census_Aggregation_distribution_tag 3
+
+#define google_census_Aggregation_interval_stats_tag 4
+#define google_census_Aggregation_name_tag       1
+#define google_census_Aggregation_description_tag 2
+#define google_census_Aggregation_tag_tag        5
+
+/* Struct field encoding specification for nanopb */
+extern const pb_field_t google_census_Duration_fields[3];
+extern const pb_field_t google_census_Timestamp_fields[3];
+extern const pb_field_t google_census_Metric_fields[5];
+extern const pb_field_t google_census_Metric_BasicUnit_fields[2];
+extern const pb_field_t google_census_Metric_MeasurementUnit_fields[4];
+extern const pb_field_t google_census_AggregationDescriptor_fields[3];
+extern const pb_field_t google_census_AggregationDescriptor_BucketBoundaries_fields[2];
+extern const pb_field_t google_census_AggregationDescriptor_IntervalBoundaries_fields[2];
+extern const pb_field_t google_census_Distribution_fields[5];
+extern const pb_field_t google_census_Distribution_Range_fields[3];
+extern const pb_field_t google_census_IntervalStats_fields[2];
+extern const pb_field_t google_census_IntervalStats_Window_fields[4];
+extern const pb_field_t google_census_Tag_fields[3];
+extern const pb_field_t google_census_View_fields[6];
+extern const pb_field_t google_census_Aggregation_fields[6];
+extern const pb_field_t google_census_ViewAggregations_fields[4];
+
+/* Maximum encoded size of messages (where known) */
+#define google_census_Duration_size              22
+#define google_census_Timestamp_size             22
+#define google_census_Metric_BasicUnit_size      2
+#define google_census_Distribution_Range_size    18
+#define google_census_IntervalStats_Window_size  44
+#define google_census_Tag_size                   516
+
+/* Message IDs (where set with "msgid" option) */
+#ifdef PB_MSGID
+
+#define CENSUS_MESSAGES \
+
+
+#endif
+
+#ifdef __cplusplus
+} /* extern "C" */
+#endif
+
+#endif
diff --git a/src/core/ext/client_config/subchannel.c b/src/core/ext/client_config/subchannel.c
index 4a463b5a8b8d99573dc7e874fe659d692c803dab..f7d98ec4d9c07a7163e6b87a1185dc4b33163a17 100644
--- a/src/core/ext/client_config/subchannel.c
+++ b/src/core/ext/client_config/subchannel.c
@@ -320,7 +320,7 @@ grpc_subchannel *grpc_subchannel_create(grpc_exec_ctx *exec_ctx,
     c->filters = NULL;
   }
   c->addr = gpr_malloc(args->addr_len);
-  memcpy(c->addr, args->addr, args->addr_len);
+  if (args->addr_len) memcpy(c->addr, args->addr, args->addr_len);
   c->pollset_set = grpc_pollset_set_create();
   c->addr_len = args->addr_len;
   grpc_set_initial_connect_string(&c->addr, &c->addr_len,
diff --git a/src/core/ext/client_config/subchannel_index.c b/src/core/ext/client_config/subchannel_index.c
index ab8d9bd91d83ab23cc952512d91feaaac6e8715a..690cb16b96f06237017603ce49ff503cde43fb76 100644
--- a/src/core/ext/client_config/subchannel_index.c
+++ b/src/core/ext/client_config/subchannel_index.c
@@ -77,12 +77,19 @@ static grpc_subchannel_key *create_key(
   grpc_subchannel_key *k = gpr_malloc(sizeof(*k));
   k->connector = grpc_connector_ref(connector);
   k->args.filter_count = args->filter_count;
-  k->args.filters = gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
-  memcpy((grpc_channel_filter *)k->args.filters, args->filters,
-         sizeof(*k->args.filters) * k->args.filter_count);
+  if (k->args.filter_count > 0) {
+    k->args.filters =
+        gpr_malloc(sizeof(*k->args.filters) * k->args.filter_count);
+    memcpy((grpc_channel_filter *)k->args.filters, args->filters,
+           sizeof(*k->args.filters) * k->args.filter_count);
+  } else {
+    k->args.filters = NULL;
+  }
   k->args.addr_len = args->addr_len;
   k->args.addr = gpr_malloc(args->addr_len);
-  memcpy(k->args.addr, args->addr, k->args.addr_len);
+  if (k->args.addr_len > 0) {
+    memcpy(k->args.addr, args->addr, k->args.addr_len);
+  }
   k->args.args = copy_channel_args(args->args);
   return k;
 }
@@ -104,11 +111,15 @@ static int subchannel_key_compare(grpc_subchannel_key *a,
   if (c != 0) return c;
   c = GPR_ICMP(a->args.filter_count, b->args.filter_count);
   if (c != 0) return c;
-  c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
-  if (c != 0) return c;
-  c = memcmp(a->args.filters, b->args.filters,
-             a->args.filter_count * sizeof(*a->args.filters));
-  if (c != 0) return c;
+  if (a->args.addr_len) {
+    c = memcmp(a->args.addr, b->args.addr, a->args.addr_len);
+    if (c != 0) return c;
+  }
+  if (a->args.filter_count > 0) {
+    c = memcmp(a->args.filters, b->args.filters,
+               a->args.filter_count * sizeof(*a->args.filters));
+    if (c != 0) return c;
+  }
   return grpc_channel_args_compare(a->args.args, b->args.args);
 }
 
diff --git a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
index d5dc39ab94f2b48ea36717c1a69f3df3f556da34..46fe588f72d930f63d3295a52a2482e3e2be9b63 100644
--- a/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
+++ b/src/core/ext/lb_policy/grpclb/proto/grpc/lb/v1/load_balancer.pb.h
@@ -33,8 +33,8 @@
 /* Automatically generated nanopb header */
 /* Generated by nanopb-0.3.5-dev */
 
-#ifndef PB_LOAD_BALANCER_PB_H_INCLUDED
-#define PB_LOAD_BALANCER_PB_H_INCLUDED
+#ifndef GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
+#define GRPC_CORE_EXT_LB_POLICY_GRPCLB_PROTO_GRPC_LB_V1_LOAD_BALANCER_PB_H
 #include "third_party/nanopb/pb.h"
 #if PB_PROTO_HEADER_VERSION != 30
 #error Regenerate this file with the current version of nanopb generator.
diff --git a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
index 58af6f995a4dc1da5465299e03790674f3c3ebf2..a262306085c7892eef0bd339ec3e778ac0d29ff7 100644
--- a/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
+++ b/src/core/ext/transport/chttp2/client/secure/secure_channel_create.c
@@ -45,9 +45,9 @@
 #include "src/core/ext/transport/chttp2/transport/chttp2_transport.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/tcp_client.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/channel.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
diff --git a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
index e21fa2a0727e8e599a19514d2a5122804316e386..0428bb1e3dc334ababf6aac943b7e339b14ea3c6 100644
--- a/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/insecure/server_chttp2.c
@@ -43,14 +43,8 @@
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *server,
-                            grpc_transport *transport) {
-  grpc_server_setup_transport(exec_ctx, server, transport,
-                              grpc_server_get_channel_args(server));
-}
-
 static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
-                          grpc_endpoint *tcp,
+                          grpc_endpoint *tcp, grpc_pollset *accepting_pollset,
                           grpc_tcp_server_acceptor *acceptor) {
   /*
    * Beware that the call to grpc_create_chttp2_transport() has to happen before
@@ -61,7 +55,8 @@ static void new_transport(grpc_exec_ctx *exec_ctx, void *server,
    */
   grpc_transport *transport = grpc_create_chttp2_transport(
       exec_ctx, grpc_server_get_channel_args(server), tcp, 0);
-  setup_transport(exec_ctx, server, transport);
+  grpc_server_setup_transport(exec_ctx, server, transport, accepting_pollset,
+                              grpc_server_get_channel_args(server));
   grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
 }
 
diff --git a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
index 698b2bef61086343efc41f33f2bb661e3e52c78c..ebbefbcd89300067045676a2335597020abcfce6 100644
--- a/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
+++ b/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.c
@@ -45,14 +45,14 @@
 #include "src/core/lib/iomgr/endpoint.h"
 #include "src/core/lib/iomgr/resolve_address.h"
 #include "src/core/lib/iomgr/tcp_server.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/server.h"
 
-typedef struct grpc_server_secure_state {
+typedef struct server_secure_state {
   grpc_server *server;
   grpc_tcp_server *tcp;
   grpc_server_security_connector *sc;
@@ -62,13 +62,16 @@ typedef struct grpc_server_secure_state {
   gpr_refcount refcount;
   grpc_closure destroy_closure;
   grpc_closure *destroy_callback;
-} grpc_server_secure_state;
+} server_secure_state;
 
-static void state_ref(grpc_server_secure_state *state) {
-  gpr_ref(&state->refcount);
-}
+typedef struct server_secure_connect {
+  server_secure_state *state;
+  grpc_pollset *accepting_pollset;
+} server_secure_connect;
+
+static void state_ref(server_secure_state *state) { gpr_ref(&state->refcount); }
 
-static void state_unref(grpc_server_secure_state *state) {
+static void state_unref(server_secure_state *state) {
   if (gpr_unref(&state->refcount)) {
     /* ensure all threads have unlocked */
     gpr_mu_lock(&state->mu);
@@ -80,67 +83,66 @@ static void state_unref(grpc_server_secure_state *state) {
   }
 }
 
-static void setup_transport(grpc_exec_ctx *exec_ctx, void *statep,
-                            grpc_transport *transport,
-                            grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
-  grpc_channel_args *args_copy;
-  grpc_arg args_to_add[2];
-  args_to_add[0] = grpc_server_credentials_to_arg(state->creds);
-  args_to_add[1] = grpc_auth_context_to_arg(auth_context);
-  args_copy = grpc_channel_args_copy_and_add(
-      grpc_server_get_channel_args(state->server), args_to_add,
-      GPR_ARRAY_SIZE(args_to_add));
-  grpc_server_setup_transport(exec_ctx, state->server, transport, args_copy);
-  grpc_channel_args_destroy(args_copy);
-}
-
 static void on_secure_handshake_done(grpc_exec_ctx *exec_ctx, void *statep,
                                      grpc_security_status status,
                                      grpc_endpoint *secure_endpoint,
                                      grpc_auth_context *auth_context) {
-  grpc_server_secure_state *state = statep;
+  server_secure_connect *state = statep;
   grpc_transport *transport;
   if (status == GRPC_SECURITY_OK) {
     if (secure_endpoint) {
-      gpr_mu_lock(&state->mu);
-      if (!state->is_shutdown) {
+      gpr_mu_lock(&state->state->mu);
+      if (!state->state->is_shutdown) {
         transport = grpc_create_chttp2_transport(
-            exec_ctx, grpc_server_get_channel_args(state->server),
+            exec_ctx, grpc_server_get_channel_args(state->state->server),
             secure_endpoint, 0);
-        setup_transport(exec_ctx, state, transport, auth_context);
+        grpc_channel_args *args_copy;
+        grpc_arg args_to_add[2];
+        args_to_add[0] = grpc_server_credentials_to_arg(state->state->creds);
+        args_to_add[1] = grpc_auth_context_to_arg(auth_context);
+        args_copy = grpc_channel_args_copy_and_add(
+            grpc_server_get_channel_args(state->state->server), args_to_add,
+            GPR_ARRAY_SIZE(args_to_add));
+        grpc_server_setup_transport(exec_ctx, state->state->server, transport,
+                                    state->accepting_pollset, args_copy);
+        grpc_channel_args_destroy(args_copy);
         grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
       } else {
         /* We need to consume this here, because the server may already have
          * gone away. */
         grpc_endpoint_destroy(exec_ctx, secure_endpoint);
       }
-      gpr_mu_unlock(&state->mu);
+      gpr_mu_unlock(&state->state->mu);
     }
   } else {
     gpr_log(GPR_ERROR, "Secure transport failed with error %d", status);
   }
-  state_unref(state);
+  state_unref(state->state);
+  gpr_free(state);
 }
 
 static void on_accept(grpc_exec_ctx *exec_ctx, void *statep, grpc_endpoint *tcp,
+                      grpc_pollset *accepting_pollset,
                       grpc_tcp_server_acceptor *acceptor) {
-  grpc_server_secure_state *state = statep;
-  state_ref(state);
-  grpc_server_security_connector_do_handshake(
-      exec_ctx, state->sc, acceptor, tcp, on_secure_handshake_done, state);
+  server_secure_connect *state = gpr_malloc(sizeof(*state));
+  state->state = statep;
+  state_ref(state->state);
+  state->accepting_pollset = accepting_pollset;
+  grpc_server_security_connector_do_handshake(exec_ctx, state->state->sc,
+                                              acceptor, tcp,
+                                              on_secure_handshake_done, state);
 }
 
 /* Server callback: start listening on our ports */
 static void start(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                   grpc_pollset **pollsets, size_t pollset_count) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server_start(exec_ctx, state->tcp, pollsets, pollset_count,
                         on_accept, state);
 }
 
 static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   if (state->destroy_callback != NULL) {
     state->destroy_callback->cb(exec_ctx, state->destroy_callback->cb_arg,
                                 success);
@@ -153,7 +155,7 @@ static void destroy_done(grpc_exec_ctx *exec_ctx, void *statep, bool success) {
    callbacks) */
 static void destroy(grpc_exec_ctx *exec_ctx, grpc_server *server, void *statep,
                     grpc_closure *callback) {
-  grpc_server_secure_state *state = statep;
+  server_secure_state *state = statep;
   grpc_tcp_server *tcp;
   gpr_mu_lock(&state->mu);
   state->is_shutdown = 1;
@@ -167,7 +169,7 @@ int grpc_server_add_secure_http2_port(grpc_server *server, const char *addr,
                                       grpc_server_credentials *creds) {
   grpc_resolved_addresses *resolved = NULL;
   grpc_tcp_server *tcp = NULL;
-  grpc_server_secure_state *state = NULL;
+  server_secure_state *state = NULL;
   size_t i;
   unsigned count = 0;
   int port_num = -1;
diff --git a/src/core/ext/transport/chttp2/transport/frame_goaway.c b/src/core/ext/transport/chttp2/transport/frame_goaway.c
index 69accb7696dd8ad93a982b379446d2b35a830974..827e7a697707f9b44d6c4d972cd6c39f3b7247f6 100644
--- a/src/core/ext/transport/chttp2/transport/frame_goaway.c
+++ b/src/core/ext/transport/chttp2/transport/frame_goaway.c
@@ -137,7 +137,8 @@ grpc_chttp2_parse_error grpc_chttp2_goaway_parser_parse(
       ++cur;
     /* fallthrough */
     case GRPC_CHTTP2_GOAWAY_DEBUG:
-      memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
+      if (end != cur)
+        memcpy(p->debug_data + p->debug_pos, cur, (size_t)(end - cur));
       GPR_ASSERT((size_t)(end - cur) < UINT32_MAX - p->debug_pos);
       p->debug_pos += (uint32_t)(end - cur);
       p->state = GRPC_CHTTP2_GOAWAY_DEBUG;
diff --git a/src/core/ext/transport/chttp2/transport/hpack_parser.c b/src/core/ext/transport/chttp2/transport/hpack_parser.c
index 687936bfd35a51f96d4749e12d3a1f6363205e69..ed45bc9cb38a9ab297ebaf774aee5fa52615bf9f 100644
--- a/src/core/ext/transport/chttp2/transport/hpack_parser.c
+++ b/src/core/ext/transport/chttp2/transport/hpack_parser.c
@@ -1138,6 +1138,7 @@ static int parse_string_prefix(grpc_chttp2_hpack_parser *p, const uint8_t *cur,
 /* append some bytes to a string */
 static void append_bytes(grpc_chttp2_hpack_parser_string *str,
                          const uint8_t *data, size_t length) {
+  if (length == 0) return;
   if (length + str->length > str->capacity) {
     GPR_ASSERT(str->length + length <= UINT32_MAX);
     str->capacity = (uint32_t)(str->length + length);
@@ -1445,6 +1446,11 @@ grpc_chttp2_parse_error grpc_chttp2_header_parser_parse(
        stream id on a header */
     if (stream_parsing != NULL) {
       if (parser->is_boundary) {
+        if (stream_parsing->header_frames_received ==
+            GPR_ARRAY_SIZE(stream_parsing->got_metadata_on_parse)) {
+          gpr_log(GPR_ERROR, "too many trailer frames");
+          return GRPC_CHTTP2_CONNECTION_ERROR;
+        }
         stream_parsing
             ->got_metadata_on_parse[stream_parsing->header_frames_received] = 1;
         stream_parsing->header_frames_received++;
diff --git a/src/core/ext/transport/cronet/transport/cronet_transport.c b/src/core/ext/transport/cronet/transport/cronet_transport.c
index 5bb085195c6abd76e18d9fdf6afd712d78bc321e..df160aaaf6186d5e187c507ff1b8ef8026da4843 100644
--- a/src/core/ext/transport/cronet/transport/cronet_transport.c
+++ b/src/core/ext/transport/cronet/transport/cronet_transport.c
@@ -218,8 +218,11 @@ static void on_write_completed(cronet_bidirectional_stream *stream,
 static void process_recv_message(stream_obj *s, const uint8_t *recv_data) {
   gpr_slice read_data_slice = gpr_slice_malloc((uint32_t)s->total_read_bytes);
   uint8_t *dst_p = GPR_SLICE_START_PTR(read_data_slice);
-  memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
-  gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  if (s->total_read_bytes > 0) {
+    // Only copy if there is non-zero number of bytes
+    memcpy(dst_p, recv_data, (size_t)s->total_read_bytes);
+    gpr_slice_buffer_add(&s->read_slice_buffer, read_data_slice);
+  }
   grpc_slice_buffer_stream_init(&s->sbs, &s->read_slice_buffer, 0);
   *s->recv_message = (grpc_byte_buffer *)&s->sbs;
 }
@@ -347,8 +350,17 @@ static void next_recv_step(stream_obj *s, enum e_caller caller) {
           if (grpc_cronet_trace) {
             gpr_log(GPR_DEBUG, "R: cronet_bidirectional_stream_read()");
           }
-          cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
-                                           s->remaining_read_bytes);
+          if (s->remaining_read_bytes > 0) {
+            cronet_bidirectional_stream_read(s->cbs, (char *)s->read_buffer,
+                                             s->remaining_read_bytes);
+          } else {
+            // Calling the closing callback directly since this is a 0 byte read
+            // for an empty message.
+            process_recv_message(s, NULL);
+            enqueue_callbacks(s->callback_list[CB_RECV_MESSAGE]);
+            invoke_closing_callback(s);
+            set_recv_state(s, CRONET_RECV_CLOSED);
+          }
         }
       }
       break;
diff --git a/src/core/lib/channel/channel_args.c b/src/core/lib/channel/channel_args.c
index 893cf0700e54ad39c6de8af8c852d94c61781061..569be4dc2825113b1119ce1f8b66116d1a9710a2 100644
--- a/src/core/lib/channel/channel_args.c
+++ b/src/core/lib/channel/channel_args.c
@@ -132,7 +132,8 @@ grpc_channel_args *grpc_channel_args_normalize(const grpc_channel_args *a) {
   for (size_t i = 0; i < a->num_args; i++) {
     args[i] = &a->args[i];
   }
-  qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
+  if (a->num_args > 1)
+    qsort(args, a->num_args, sizeof(grpc_arg *), cmp_key_stable);
 
   grpc_channel_args *b = gpr_malloc(sizeof(grpc_channel_args));
   b->num_args = a->num_args;
diff --git a/src/core/lib/compression/compression_algorithm.c b/src/core/lib/compression/compression_algorithm.c
index 7039364b7bcbb5b2c0896802b047aa6d6b1eee41..820871d579bf590f3691e0bb4c9729fcfd54fa64 100644
--- a/src/core/lib/compression/compression_algorithm.c
+++ b/src/core/lib/compression/compression_algorithm.c
@@ -199,5 +199,6 @@ void grpc_compression_options_disable_algorithm(
 int grpc_compression_options_is_algorithm_enabled(
     const grpc_compression_options *opts,
     grpc_compression_algorithm algorithm) {
+  if (algorithm >= GRPC_COMPRESS_ALGORITHMS_COUNT) return 0;
   return GPR_BITGET(opts->enabled_algorithms_bitset, algorithm);
 }
diff --git a/src/core/lib/http/httpcli_security_connector.c b/src/core/lib/http/httpcli_security_connector.c
index ea4bff30d4b245a052d3c9f569a88f4fc0436ebe..559092896849b44c70953cc92caa869c300e30f2 100644
--- a/src/core/lib/http/httpcli_security_connector.c
+++ b/src/core/lib/http/httpcli_security_connector.c
@@ -38,7 +38,7 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/tsi/ssl_transport_security.h"
 
diff --git a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
index aeb6e286658b113e672883c01d2ce45a7b6438d8..943c404f91712e17a8d2cc516f4d99323d6fa6ee 100644
--- a/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_and_epoll_posix.c
@@ -126,6 +126,9 @@ struct grpc_fd {
   grpc_closure *on_done_closure;
 
   grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
 };
 
 /* Begin polling on an fd.
@@ -147,7 +150,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
    if got_read or got_write are 1, also does the become_{readable,writable} as
    appropriate. */
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
 static bool fd_is_orphaned(grpc_fd *fd);
@@ -342,6 +346,7 @@ static grpc_fd *alloc_fd(int fd) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  r->read_notifier_pollset = NULL;
   gpr_mu_unlock(&r->mu);
   return r;
 }
@@ -545,6 +550,11 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   }
 }
 
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
   GPR_ASSERT(!fd->shutdown);
@@ -568,6 +578,18 @@ static void fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   gpr_mu_unlock(&fd->mu);
 }
 
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
 static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
                               grpc_pollset_worker *worker, uint32_t read_mask,
                               uint32_t write_mask, grpc_fd_watcher *watcher) {
@@ -620,7 +642,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
 }
 
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
   int was_polling = 0;
   int kick = 0;
   grpc_fd *fd = watcher->fd;
@@ -656,6 +679,10 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
     if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
       kick = 1;
     }
+
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
   }
   if (got_write) {
     if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
@@ -756,9 +783,14 @@ static void pollset_kick_ext(grpc_pollset *p,
     specific_worker = pop_front_worker(p);
     if (specific_worker != NULL) {
       if (gpr_tls_get(&g_current_thread_worker) == (intptr_t)specific_worker) {
+        /* Prefer not to kick self. Push the worker to the end of the list and
+         * pop the one from front */
         GPR_TIMER_MARK("kick_anonymous_not_self", 0);
         push_back_worker(p, specific_worker);
         specific_worker = pop_front_worker(p);
+        /* If there was only one worker on the pollset, we would get the same
+         * worker we pushed (the one set on current thread local) back. If so,
+         * kick it only if GRPC_POLLSET_CAN_KICK_SELF flag is set */
         if ((flags & GRPC_POLLSET_CAN_KICK_SELF) == 0 &&
             gpr_tls_get(&g_current_thread_worker) ==
                 (intptr_t)specific_worker) {
@@ -1201,11 +1233,11 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
       gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
     }
     if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   } else if (r == 0) {
     if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   } else {
     if (pfd[0].revents & POLLIN_CHECK) {
@@ -1216,9 +1248,9 @@ static void basic_pollset_maybe_work_and_unlock(grpc_exec_ctx *exec_ctx,
     }
     if (nfds > 2) {
       fd_end_poll(exec_ctx, &fd_watcher, pfd[2].revents & POLLIN_CHECK,
-                  pfd[2].revents & POLLOUT_CHECK);
+                  pfd[2].revents & POLLOUT_CHECK, pollset);
     } else if (fd) {
-      fd_end_poll(exec_ctx, &fd_watcher, 0, 0);
+      fd_end_poll(exec_ctx, &fd_watcher, 0, 0, NULL);
     }
   }
 
@@ -1354,11 +1386,11 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock(
       gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
     }
     for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
     }
   } else if (r == 0) {
     for (i = 2; i < pfd_count; i++) {
-      fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+      fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
     }
   } else {
     if (pfds[0].revents & POLLIN_CHECK) {
@@ -1369,11 +1401,11 @@ static void multipoll_with_poll_pollset_maybe_work_and_unlock(
     }
     for (i = 2; i < pfd_count; i++) {
       if (watchers[i].fd == NULL) {
-        fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+        fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         continue;
       }
       fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                  pfds[i].revents & POLLOUT_CHECK);
+                  pfds[i].revents & POLLOUT_CHECK, pollset);
     }
   }
 
@@ -1449,20 +1481,31 @@ static void poll_become_multipoller(grpc_exec_ctx *exec_ctx,
 #include "src/core/lib/profiling/timers.h"
 #include "src/core/lib/support/block_annotate.h"
 
-static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st) {
+static void set_ready(grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_closure **st,
+                      grpc_pollset *read_notifier_pollset) {
   /* only one set_ready can be active at once (but there may be a racing
      notify_on) */
   gpr_mu_lock(&fd->mu);
   set_ready_locked(exec_ctx, fd, st);
+
+  /* A non-NULL read_notifier_pollset means that the fd is readable. */
+  if (read_notifier_pollset != NULL) {
+    /* Note: Since the fd might be a part of multiple pollsets, this might be
+     * called multiple times (for each time the fd becomes readable) and it is
+     * okay to set the fd's read-notifier pollset to anyone of these pollsets */
+    set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+  }
+
   gpr_mu_unlock(&fd->mu);
 }
 
-static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->read_closure);
+static void fd_become_readable(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
+                               grpc_pollset *notifier_pollset) {
+  set_ready(exec_ctx, fd, &fd->read_closure, notifier_pollset);
 }
 
 static void fd_become_writable(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
-  set_ready(exec_ctx, fd, &fd->write_closure);
+  set_ready(exec_ctx, fd, &fd->write_closure, NULL);
 }
 
 struct epoll_fd_list {
@@ -1554,7 +1597,7 @@ static void finally_add_fd(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       }
     }
   }
-  fd_end_poll(exec_ctx, &watcher, 0, 0);
+  fd_end_poll(exec_ctx, &watcher, 0, 0, NULL);
 }
 
 static void perform_delayed_add(grpc_exec_ctx *exec_ctx, void *arg,
@@ -1668,7 +1711,7 @@ static void multipoll_with_epoll_pollset_maybe_work_and_unlock(
               grpc_wakeup_fd_consume_wakeup(&grpc_global_wakeup_fd);
             } else {
               if (read_ev || cancel) {
-                fd_become_readable(exec_ctx, fd);
+                fd_become_readable(exec_ctx, fd, pollset);
               }
               if (write_ev || cancel) {
                 fd_become_writable(exec_ctx, fd);
@@ -1897,6 +1940,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_shutdown = fd_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,
diff --git a/src/core/lib/iomgr/ev_poll_posix.c b/src/core/lib/iomgr/ev_poll_posix.c
index e91ae4021221f1906e8221d293d9621ad8fbdfc3..0167999dada4e3c87b087b5200d8b263b8ca4ef0 100644
--- a/src/core/lib/iomgr/ev_poll_posix.c
+++ b/src/core/lib/iomgr/ev_poll_posix.c
@@ -113,6 +113,9 @@ struct grpc_fd {
   grpc_closure *on_done_closure;
 
   grpc_iomgr_object iomgr_object;
+
+  /* The pollset that last noticed and notified that the fd is readable */
+  grpc_pollset *read_notifier_pollset;
 };
 
 /* Begin polling on an fd.
@@ -134,7 +137,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
    if got_read or got_write are 1, also does the become_{readable,writable} as
    appropriate. */
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *rec,
-                        int got_read, int got_write);
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset);
 
 /* Return 1 if this fd is orphaned, 0 otherwise */
 static bool fd_is_orphaned(grpc_fd *fd);
@@ -177,7 +181,6 @@ struct grpc_pollset_worker {
 struct grpc_pollset {
   gpr_mu mu;
   grpc_pollset_worker root_worker;
-  int in_flight_cbs;
   int shutting_down;
   int called_shutdown;
   int kicked_without_pollers;
@@ -187,10 +190,6 @@ struct grpc_pollset {
   size_t fd_count;
   size_t fd_capacity;
   grpc_fd **fds;
-  /* fds that have been removed from the pollset explicitly */
-  size_t del_count;
-  size_t del_capacity;
-  grpc_fd **dels;
   /* Local cache of eventfds for workers */
   grpc_cached_wakeup_fd *local_wakeup_cache;
 };
@@ -301,6 +300,7 @@ static grpc_fd *fd_create(int fd, const char *name) {
   r->on_done_closure = NULL;
   r->closed = 0;
   r->released = 0;
+  r->read_notifier_pollset = NULL;
 
   char *name2;
   gpr_asprintf(&name2, "%s fd=%d", name, fd);
@@ -316,6 +316,18 @@ static bool fd_is_orphaned(grpc_fd *fd) {
   return (gpr_atm_acq_load(&fd->refst) & 1) == 0;
 }
 
+/* Return the read-notifier pollset */
+static grpc_pollset *fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                  grpc_fd *fd) {
+  grpc_pollset *notifier = NULL;
+
+  gpr_mu_lock(&fd->mu);
+  notifier = fd->read_notifier_pollset;
+  gpr_mu_unlock(&fd->mu);
+
+  return notifier;
+}
+
 static void pollset_kick_locked(grpc_fd_watcher *watcher) {
   gpr_mu_lock(&watcher->pollset->mu);
   GPR_ASSERT(watcher->worker);
@@ -444,6 +456,11 @@ static int set_ready_locked(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   }
 }
 
+static void set_read_notifier_pollset_locked(
+    grpc_exec_ctx *exec_ctx, grpc_fd *fd, grpc_pollset *read_notifier_pollset) {
+  fd->read_notifier_pollset = read_notifier_pollset;
+}
+
 static void fd_shutdown(grpc_exec_ctx *exec_ctx, grpc_fd *fd) {
   gpr_mu_lock(&fd->mu);
   GPR_ASSERT(!fd->shutdown);
@@ -519,7 +536,8 @@ static uint32_t fd_begin_poll(grpc_fd *fd, grpc_pollset *pollset,
 }
 
 static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
-                        int got_read, int got_write) {
+                        int got_read, int got_write,
+                        grpc_pollset *read_notifier_pollset) {
   int was_polling = 0;
   int kick = 0;
   grpc_fd *fd = watcher->fd;
@@ -555,6 +573,9 @@ static void fd_end_poll(grpc_exec_ctx *exec_ctx, grpc_fd_watcher *watcher,
     if (set_ready_locked(exec_ctx, fd, &fd->read_closure)) {
       kick = 1;
     }
+    if (read_notifier_pollset != NULL) {
+      set_read_notifier_pollset_locked(exec_ctx, fd, read_notifier_pollset);
+    }
   }
   if (got_write) {
     if (set_ready_locked(exec_ctx, fd, &fd->write_closure)) {
@@ -700,7 +721,6 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   gpr_mu_init(&pollset->mu);
   *mu = &pollset->mu;
   pollset->root_worker.next = pollset->root_worker.prev = &pollset->root_worker;
-  pollset->in_flight_cbs = 0;
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -709,14 +729,10 @@ static void pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
   pollset->kicked_without_pollers = 0;
   pollset->fd_count = 0;
   pollset->fd_capacity = 0;
-  pollset->del_count = 0;
-  pollset->del_capacity = 0;
   pollset->fds = NULL;
-  pollset->dels = NULL;
 }
 
 static void pollset_destroy(grpc_pollset *pollset) {
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   while (pollset->local_wakeup_cache) {
@@ -726,17 +742,14 @@ static void pollset_destroy(grpc_pollset *pollset) {
     pollset->local_wakeup_cache = next;
   }
   gpr_free(pollset->fds);
-  gpr_free(pollset->dels);
   gpr_mu_destroy(&pollset->mu);
 }
 
 static void pollset_reset(grpc_pollset *pollset) {
   GPR_ASSERT(pollset->shutting_down);
-  GPR_ASSERT(pollset->in_flight_cbs == 0);
   GPR_ASSERT(!pollset_has_workers(pollset));
   GPR_ASSERT(pollset->idle_jobs.head == pollset->idle_jobs.tail);
   GPR_ASSERT(pollset->fd_count == 0);
-  GPR_ASSERT(pollset->del_count == 0);
   pollset->shutting_down = 0;
   pollset->called_shutdown = 0;
   pollset->kicked_without_pollers = 0;
@@ -769,11 +782,7 @@ static void finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset) {
   for (i = 0; i < pollset->fd_count; i++) {
     GRPC_FD_UNREF(pollset->fds[i], "multipoller");
   }
-  for (i = 0; i < pollset->del_count; i++) {
-    GRPC_FD_UNREF(pollset->dels[i], "multipoller_del");
-  }
   pollset->fd_count = 0;
-  pollset->del_count = 0;
   grpc_exec_ctx_enqueue(exec_ctx, pollset->shutdown_done, true, NULL);
 }
 
@@ -813,13 +822,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
     GPR_TIMER_MARK("pollset_work.shutting_down", 0);
     goto done;
   }
-  /* Give do_promote priority so we don't starve it out */
-  if (pollset->in_flight_cbs) {
-    GPR_TIMER_MARK("pollset_work.in_flight_cbs", 0);
-    gpr_mu_unlock(&pollset->mu);
-    locked = 0;
-    goto done;
-  }
   /* Start polling, and keep doing so while we're being asked to
      re-evaluate our pollers (this allows poll() based pollers to
      ensure they don't miss wakeups) */
@@ -839,7 +841,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
 
       int timeout;
       int r;
-      size_t i, j, fd_count;
+      size_t i, fd_count;
       nfds_t pfd_count;
       /* TODO(ctiller): inline some elements to avoid an allocation */
       grpc_fd_watcher *watchers;
@@ -859,11 +861,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
       pfds[1].events = POLLIN;
       pfds[1].revents = 0;
       for (i = 0; i < pollset->fd_count; i++) {
-        int remove = fd_is_orphaned(pollset->fds[i]);
-        for (j = 0; !remove && j < pollset->del_count; j++) {
-          if (pollset->fds[i] == pollset->dels[j]) remove = 1;
-        }
-        if (remove) {
+        if (fd_is_orphaned(pollset->fds[i])) {
           GRPC_FD_UNREF(pollset->fds[i], "multipoller");
         } else {
           pollset->fds[fd_count++] = pollset->fds[i];
@@ -874,10 +872,6 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           pfd_count++;
         }
       }
-      for (j = 0; j < pollset->del_count; j++) {
-        GRPC_FD_UNREF(pollset->dels[j], "multipoller_del");
-      }
-      pollset->del_count = 0;
       pollset->fd_count = fd_count;
       gpr_mu_unlock(&pollset->mu);
 
@@ -899,11 +893,11 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
           gpr_log(GPR_ERROR, "poll() failed: %s", strerror(errno));
         }
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else if (r == 0) {
         for (i = 2; i < pfd_count; i++) {
-          fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+          fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
         }
       } else {
         if (pfds[0].revents & POLLIN_CHECK) {
@@ -914,10 +908,10 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
         }
         for (i = 2; i < pfd_count; i++) {
           if (watchers[i].fd == NULL) {
-            fd_end_poll(exec_ctx, &watchers[i], 0, 0);
+            fd_end_poll(exec_ctx, &watchers[i], 0, 0, NULL);
           } else {
             fd_end_poll(exec_ctx, &watchers[i], pfds[i].revents & POLLIN_CHECK,
-                        pfds[i].revents & POLLOUT_CHECK);
+                        pfds[i].revents & POLLOUT_CHECK, pollset);
           }
         }
       }
@@ -969,7 +963,7 @@ static void pollset_work(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (pollset->shutting_down) {
     if (pollset_has_workers(pollset)) {
       pollset_kick(pollset, NULL);
-    } else if (!pollset->called_shutdown && pollset->in_flight_cbs == 0) {
+    } else if (!pollset->called_shutdown) {
       pollset->called_shutdown = 1;
       gpr_mu_unlock(&pollset->mu);
       finish_shutdown(exec_ctx, pollset);
@@ -999,8 +993,7 @@ static void pollset_shutdown(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
   if (!pollset_has_workers(pollset)) {
     grpc_exec_ctx_enqueue_list(exec_ctx, &pollset->idle_jobs, NULL);
   }
-  if (!pollset->called_shutdown && pollset->in_flight_cbs == 0 &&
-      !pollset_has_workers(pollset)) {
+  if (!pollset->called_shutdown && !pollset_has_workers(pollset)) {
     pollset->called_shutdown = 1;
     finish_shutdown(exec_ctx, pollset);
   }
@@ -1181,6 +1174,7 @@ static const grpc_event_engine_vtable vtable = {
     .fd_shutdown = fd_shutdown,
     .fd_notify_on_read = fd_notify_on_read,
     .fd_notify_on_write = fd_notify_on_write,
+    .fd_get_read_notifier_pollset = fd_get_read_notifier_pollset,
 
     .pollset_init = pollset_init,
     .pollset_shutdown = pollset_shutdown,
diff --git a/src/core/lib/iomgr/ev_posix.c b/src/core/lib/iomgr/ev_posix.c
index a7dfc9552d5ab47b370a842a73fb124f3b29d843..6477b05dcd8c58af77d9833e9cb84c87e9aefce5 100644
--- a/src/core/lib/iomgr/ev_posix.c
+++ b/src/core/lib/iomgr/ev_posix.c
@@ -163,6 +163,11 @@ void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
   g_event_engine->fd_notify_on_write(exec_ctx, fd, closure);
 }
 
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd) {
+  return g_event_engine->fd_get_read_notifier_pollset(exec_ctx, fd);
+}
+
 size_t grpc_pollset_size(void) { return g_event_engine->pollset_size; }
 
 void grpc_pollset_init(grpc_pollset *pollset, gpr_mu **mu) {
diff --git a/src/core/lib/iomgr/ev_posix.h b/src/core/lib/iomgr/ev_posix.h
index 1fa9f5ef2d69f2c71706ab0b51987d3d05203d24..344bf63438aaa281a386e1bac4e54529044aba24 100644
--- a/src/core/lib/iomgr/ev_posix.h
+++ b/src/core/lib/iomgr/ev_posix.h
@@ -55,6 +55,8 @@ typedef struct grpc_event_engine_vtable {
                             grpc_closure *closure);
   void (*fd_notify_on_write)(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
+  grpc_pollset *(*fd_get_read_notifier_pollset)(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
 
   void (*pollset_init)(grpc_pollset *pollset, gpr_mu **mu);
   void (*pollset_shutdown)(grpc_exec_ctx *exec_ctx, grpc_pollset *pollset,
@@ -137,6 +139,10 @@ void grpc_fd_notify_on_read(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
 void grpc_fd_notify_on_write(grpc_exec_ctx *exec_ctx, grpc_fd *fd,
                              grpc_closure *closure);
 
+/* Return the read notifier pollset from the fd */
+grpc_pollset *grpc_fd_get_read_notifier_pollset(grpc_exec_ctx *exec_ctx,
+                                                grpc_fd *fd);
+
 /* pollset_posix functions */
 
 /* Add an fd to a pollset */
diff --git a/src/core/lib/iomgr/tcp_server.h b/src/core/lib/iomgr/tcp_server.h
index 99b9f297291e3c9f2710ab379878442d2df04d09..fee14ae661794d50de325ede858339c6f900d72a 100644
--- a/src/core/lib/iomgr/tcp_server.h
+++ b/src/core/lib/iomgr/tcp_server.h
@@ -52,6 +52,7 @@ typedef struct grpc_tcp_server_acceptor {
 /* Called for newly connected TCP connections. */
 typedef void (*grpc_tcp_server_cb)(grpc_exec_ctx *exec_ctx, void *arg,
                                    grpc_endpoint *ep,
+                                   grpc_pollset *accepting_pollset,
                                    grpc_tcp_server_acceptor *acceptor);
 
 /* Create a server, initially not bound to any ports. The caller owns one ref.
diff --git a/src/core/lib/iomgr/tcp_server_posix.c b/src/core/lib/iomgr/tcp_server_posix.c
index aaeb384f6e4b43c80df5ba429c94c13f490e2df4..909e34abc70e1d5cbcef77e1d9a9ad6eb9236eee 100644
--- a/src/core/lib/iomgr/tcp_server_posix.c
+++ b/src/core/lib/iomgr/tcp_server_posix.c
@@ -128,6 +128,9 @@ struct grpc_tcp_server {
   grpc_pollset **pollsets;
   /* number of pollsets in the pollsets array */
   size_t pollset_count;
+
+  /* next pollset to assign a channel to */
+  size_t next_pollset_to_assign;
 };
 
 grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
@@ -145,6 +148,7 @@ grpc_tcp_server *grpc_tcp_server_create(grpc_closure *shutdown_complete) {
   s->head = NULL;
   s->tail = NULL;
   s->nports = 0;
+  s->next_pollset_to_assign = 0;
   return s;
 }
 
@@ -310,13 +314,17 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   grpc_tcp_listener *sp = arg;
   grpc_tcp_server_acceptor acceptor = {sp->server, sp->port_index,
                                        sp->fd_index};
+  grpc_pollset *read_notifier_pollset = NULL;
   grpc_fd *fdobj;
-  size_t i;
 
   if (!success) {
     goto error;
   }
 
+  read_notifier_pollset =
+      sp->server->pollsets[(sp->server->next_pollset_to_assign++) %
+                           sp->server->pollset_count];
+
   /* loop until accept4 returns EAGAIN, and then re-arm notification */
   for (;;) {
     struct sockaddr_storage addr;
@@ -349,16 +357,18 @@ static void on_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     }
 
     fdobj = grpc_fd_create(fd, name);
-    /* TODO(ctiller): revise this when we have server-side sharding
-       of channels -- we certainly should not be automatically adding every
-       incoming channel to every pollset owned by the server */
-    for (i = 0; i < sp->server->pollset_count; i++) {
-      grpc_pollset_add_fd(exec_ctx, sp->server->pollsets[i], fdobj);
+
+    if (read_notifier_pollset == NULL) {
+      gpr_log(GPR_ERROR, "Read notifier pollset is not set on the fd");
+      goto error;
     }
+
+    grpc_pollset_add_fd(exec_ctx, read_notifier_pollset, fdobj);
+
     sp->server->on_accept_cb(
         exec_ctx, sp->server->on_accept_cb_arg,
         grpc_tcp_create(fdobj, GRPC_TCP_DEFAULT_READ_SLICE_SIZE, addr_str),
-        &acceptor);
+        read_notifier_pollset, &acceptor);
 
     gpr_free(name);
     gpr_free(addr_str);
diff --git a/src/core/lib/iomgr/tcp_server_windows.c b/src/core/lib/iomgr/tcp_server_windows.c
index 125f521d87e556b50a400d0cfdc943221d97242c..e15f8b0cdf209e81e43288ac10d03caccd38a802 100644
--- a/src/core/lib/iomgr/tcp_server_windows.c
+++ b/src/core/lib/iomgr/tcp_server_windows.c
@@ -379,9 +379,10 @@ static void on_accept(grpc_exec_ctx *exec_ctx, void *arg, bool from_iocp) {
 
   /* The only time we should call our callback, is where we successfully
      managed to accept a connection, and created an endpoint. */
-  if (ep)
-    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep,
+  if (ep) {
+    sp->server->on_accept_cb(exec_ctx, sp->server->on_accept_cb_arg, ep, NULL,
                              &acceptor);
+  }
   /* As we were notified from the IOCP of one and exactly one accept,
      the former socked we created has now either been destroy or assigned
      to the new connection. We need to create a new one for the next
diff --git a/src/core/lib/security/security_context.c b/src/core/lib/security/context/security_context.c
similarity index 99%
rename from src/core/lib/security/security_context.c
rename to src/core/lib/security/context/security_context.c
index 343e0b5b8b11cf1eef285497e6c112c70ed5f997..127b13ee50382a5faf45a113b98107086a0397b7 100644
--- a/src/core/lib/security/security_context.c
+++ b/src/core/lib/security/context/security_context.c
@@ -33,7 +33,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/api_trace.h"
 #include "src/core/lib/surface/call.h"
diff --git a/src/core/lib/security/security_context.h b/src/core/lib/security/context/security_context.h
similarity index 94%
rename from src/core/lib/security/security_context.h
rename to src/core/lib/security/context/security_context.h
index 81161ec47dee41c1d66d13308d0470533093a7b0..ef0c06b1fb65842d3a814979e3f34595b5dfb343 100644
--- a/src/core/lib/security/security_context.h
+++ b/src/core/lib/security/context/security_context.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H
+#ifndef GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
+#define GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H
 
 #include "src/core/lib/iomgr/pollset.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 /* --- grpc_auth_context ---
 
@@ -111,4 +111,4 @@ grpc_auth_context *grpc_auth_context_from_arg(const grpc_arg *arg);
 grpc_auth_context *grpc_find_auth_context_in_args(
     const grpc_channel_args *args);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONTEXT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CONTEXT_SECURITY_CONTEXT_H */
diff --git a/src/core/lib/security/credentials.c b/src/core/lib/security/credentials.c
deleted file mode 100644
index fd5ad3589b78ddf44692aaa652cbe05154e932a5..0000000000000000000000000000000000000000
--- a/src/core/lib/security/credentials.c
+++ /dev/null
@@ -1,1296 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include "src/core/lib/security/credentials.h"
-
-#include <stdio.h>
-#include <string.h>
-
-#include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/channel/http_client_filter.h"
-#include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/http/parser.h"
-#include "src/core/lib/iomgr/executor.h"
-#include "src/core/lib/json/json.h"
-#include "src/core/lib/support/string.h"
-#include "src/core/lib/surface/api_trace.h"
-
-#include <grpc/support/alloc.h>
-#include <grpc/support/log.h>
-#include <grpc/support/string_util.h>
-#include <grpc/support/sync.h>
-#include <grpc/support/time.h>
-
-/* -- Common. -- */
-
-struct grpc_credentials_metadata_request {
-  grpc_call_credentials *creds;
-  grpc_credentials_metadata_cb cb;
-  void *user_data;
-};
-
-static grpc_credentials_metadata_request *
-grpc_credentials_metadata_request_create(grpc_call_credentials *creds,
-                                         grpc_credentials_metadata_cb cb,
-                                         void *user_data) {
-  grpc_credentials_metadata_request *r =
-      gpr_malloc(sizeof(grpc_credentials_metadata_request));
-  r->creds = grpc_call_credentials_ref(creds);
-  r->cb = cb;
-  r->user_data = user_data;
-  return r;
-}
-
-static void grpc_credentials_metadata_request_destroy(
-    grpc_credentials_metadata_request *r) {
-  grpc_call_credentials_unref(r->creds);
-  gpr_free(r);
-}
-
-grpc_channel_credentials *grpc_channel_credentials_ref(
-    grpc_channel_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
-  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
-  grpc_channel_credentials_unref(creds);
-}
-
-grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_call_credentials_unref(grpc_call_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    gpr_free(creds);
-  }
-}
-
-void grpc_call_credentials_release(grpc_call_credentials *creds) {
-  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
-  grpc_call_credentials_unref(creds);
-}
-
-void grpc_call_credentials_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
-    if (cb != NULL) {
-      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-    }
-    return;
-  }
-  creds->vtable->get_request_metadata(exec_ctx, creds, pollset, context, cb,
-                                      user_data);
-}
-
-grpc_security_status grpc_channel_credentials_create_security_connector(
-    grpc_channel_credentials *channel_creds, const char *target,
-    const grpc_channel_args *args, grpc_channel_security_connector **sc,
-    grpc_channel_args **new_args) {
-  *new_args = NULL;
-  if (channel_creds == NULL) {
-    return GRPC_SECURITY_ERROR;
-  }
-  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
-  return channel_creds->vtable->create_security_connector(
-      channel_creds, NULL, target, args, sc, new_args);
-}
-
-grpc_server_credentials *grpc_server_credentials_ref(
-    grpc_server_credentials *creds) {
-  if (creds == NULL) return NULL;
-  gpr_ref(&creds->refcount);
-  return creds;
-}
-
-void grpc_server_credentials_unref(grpc_server_credentials *creds) {
-  if (creds == NULL) return;
-  if (gpr_unref(&creds->refcount)) {
-    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
-    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-      creds->processor.destroy(creds->processor.state);
-    }
-    gpr_free(creds);
-  }
-}
-
-void grpc_server_credentials_release(grpc_server_credentials *creds) {
-  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
-  grpc_server_credentials_unref(creds);
-}
-
-grpc_security_status grpc_server_credentials_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
-    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
-    return GRPC_SECURITY_ERROR;
-  }
-  return creds->vtable->create_security_connector(creds, sc);
-}
-
-void grpc_server_credentials_set_auth_metadata_processor(
-    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
-  GRPC_API_TRACE(
-      "grpc_server_credentials_set_auth_metadata_processor("
-      "creds=%p, "
-      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
-      3, (creds, (void *)(intptr_t)processor.process, processor.state));
-  if (creds == NULL) return;
-  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
-    creds->processor.destroy(creds->processor.state);
-  }
-  creds->processor = processor;
-}
-
-static void server_credentials_pointer_arg_destroy(void *p) {
-  grpc_server_credentials_unref(p);
-}
-
-static void *server_credentials_pointer_arg_copy(void *p) {
-  return grpc_server_credentials_ref(p);
-}
-
-static int server_credentials_pointer_cmp(void *a, void *b) {
-  return GPR_ICMP(a, b);
-}
-
-static const grpc_arg_pointer_vtable cred_ptr_vtable = {
-    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
-    server_credentials_pointer_cmp};
-
-grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
-  grpc_arg arg;
-  memset(&arg, 0, sizeof(grpc_arg));
-  arg.type = GRPC_ARG_POINTER;
-  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
-  arg.value.pointer.p = p;
-  arg.value.pointer.vtable = &cred_ptr_vtable;
-  return arg;
-}
-
-grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
-  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
-  if (arg->type != GRPC_ARG_POINTER) {
-    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
-            GRPC_SERVER_CREDENTIALS_ARG);
-    return NULL;
-  }
-  return arg->value.pointer.p;
-}
-
-grpc_server_credentials *grpc_find_server_credentials_in_args(
-    const grpc_channel_args *args) {
-  size_t i;
-  if (args == NULL) return NULL;
-  for (i = 0; i < args->num_args; i++) {
-    grpc_server_credentials *p =
-        grpc_server_credentials_from_arg(&args->args[i]);
-    if (p != NULL) return p;
-  }
-  return NULL;
-}
-
-/* -- Ssl credentials. -- */
-
-static void ssl_destruct(grpc_channel_credentials *creds) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
-  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
-}
-
-static void ssl_server_destruct(grpc_server_credentials *creds) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
-    if (c->config.pem_private_keys[i] != NULL) {
-      gpr_free(c->config.pem_private_keys[i]);
-    }
-    if (c->config.pem_cert_chains[i] != NULL) {
-      gpr_free(c->config.pem_cert_chains[i]);
-    }
-  }
-  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
-  if (c->config.pem_private_keys_sizes != NULL) {
-    gpr_free(c->config.pem_private_keys_sizes);
-  }
-  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
-  if (c->config.pem_cert_chains_sizes != NULL) {
-    gpr_free(c->config.pem_cert_chains_sizes);
-  }
-  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
-}
-
-static grpc_security_status ssl_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_OK;
-  size_t i = 0;
-  const char *overridden_target_name = NULL;
-  grpc_arg new_arg;
-
-  for (i = 0; args && i < args->num_args; i++) {
-    grpc_arg *arg = &args->args[i];
-    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
-        arg->type == GRPC_ARG_STRING) {
-      overridden_target_name = arg->value.string;
-      break;
-    }
-  }
-  status = grpc_ssl_channel_security_connector_create(
-      call_creds, &c->config, target, overridden_target_name, sc);
-  if (status != GRPC_SECURITY_OK) {
-    return status;
-  }
-  new_arg.type = GRPC_ARG_STRING;
-  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
-  new_arg.value.string = "https";
-  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
-  return status;
-}
-
-static grpc_security_status ssl_server_create_security_connector(
-    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
-  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
-  return grpc_ssl_server_security_connector_create(&c->config, sc);
-}
-
-static grpc_channel_credentials_vtable ssl_vtable = {
-    ssl_destruct, ssl_create_security_connector};
-
-static grpc_server_credentials_vtable ssl_server_vtable = {
-    ssl_server_destruct, ssl_server_create_security_connector};
-
-static void ssl_copy_key_material(const char *input, unsigned char **output,
-                                  size_t *output_size) {
-  *output_size = strlen(input);
-  *output = gpr_malloc(*output_size);
-  memcpy(*output, input, *output_size);
-}
-
-static void ssl_build_config(const char *pem_root_certs,
-                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-                             grpc_ssl_config *config) {
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (pem_key_cert_pair != NULL) {
-    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pair->private_key,
-                          &config->pem_private_key,
-                          &config->pem_private_key_size);
-    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
-                          &config->pem_cert_chain,
-                          &config->pem_cert_chain_size);
-  }
-}
-
-static void ssl_build_server_config(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    grpc_ssl_server_config *config) {
-  size_t i;
-  config->client_certificate_request = client_certificate_request;
-  if (pem_root_certs != NULL) {
-    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
-                          &config->pem_root_certs_size);
-  }
-  if (num_key_cert_pairs > 0) {
-    GPR_ASSERT(pem_key_cert_pairs != NULL);
-    config->pem_private_keys =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_cert_chains =
-        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
-    config->pem_private_keys_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-    config->pem_cert_chains_sizes =
-        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
-  }
-  config->num_key_cert_pairs = num_key_cert_pairs;
-  for (i = 0; i < num_key_cert_pairs; i++) {
-    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
-    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
-    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
-                          &config->pem_private_keys[i],
-                          &config->pem_private_keys_sizes[i]);
-    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
-                          &config->pem_cert_chains[i],
-                          &config->pem_cert_chains_sizes[i]);
-  }
-}
-
-grpc_channel_credentials *grpc_ssl_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
-    void *reserved) {
-  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_credentials_create(pem_root_certs=%s, "
-      "pem_key_cert_pair=%p, "
-      "reserved=%p)",
-      3, (pem_root_certs, pem_key_cert_pair, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  c->base.vtable = &ssl_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
-  return &c->base;
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
-  return grpc_ssl_server_credentials_create_ex(
-      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
-      force_client_auth
-          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
-          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
-      reserved);
-}
-
-grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
-    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
-    size_t num_key_cert_pairs,
-    grpc_ssl_client_certificate_request_type client_certificate_request,
-    void *reserved) {
-  grpc_ssl_server_credentials *c =
-      gpr_malloc(sizeof(grpc_ssl_server_credentials));
-  GRPC_API_TRACE(
-      "grpc_ssl_server_credentials_create_ex("
-      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
-      "client_certificate_request=%d, reserved=%p)",
-      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
-          client_certificate_request, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_ssl_server_credentials));
-  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &ssl_server_vtable;
-  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
-                          num_key_cert_pairs, client_certificate_request,
-                          &c->config);
-  return &c->base;
-}
-
-/* -- Jwt credentials -- */
-
-static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
-  if (c->cached.jwt_md != NULL) {
-    grpc_credentials_md_store_unref(c->cached.jwt_md);
-    c->cached.jwt_md = NULL;
-  }
-  if (c->cached.service_url != NULL) {
-    gpr_free(c->cached.service_url);
-    c->cached.service_url = NULL;
-  }
-  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-}
-
-static void jwt_destruct(grpc_call_credentials *creds) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  grpc_auth_json_key_destruct(&c->key);
-  jwt_reset_cache(c);
-  gpr_mu_destroy(&c->cache_mu);
-}
-
-static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_service_account_jwt_access_credentials *c =
-      (grpc_service_account_jwt_access_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-
-  /* See if we can return a cached jwt. */
-  grpc_credentials_md_store *jwt_md = NULL;
-  {
-    gpr_mu_lock(&c->cache_mu);
-    if (c->cached.service_url != NULL &&
-        strcmp(c->cached.service_url, context.service_url) == 0 &&
-        c->cached.jwt_md != NULL &&
-        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
-                                   gpr_now(GPR_CLOCK_REALTIME)),
-                      refresh_threshold) > 0)) {
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md == NULL) {
-    char *jwt = NULL;
-    /* Generate a new jwt. */
-    gpr_mu_lock(&c->cache_mu);
-    jwt_reset_cache(c);
-    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
-                                   c->jwt_lifetime, NULL);
-    if (jwt != NULL) {
-      char *md_value;
-      gpr_asprintf(&md_value, "Bearer %s", jwt);
-      gpr_free(jwt);
-      c->cached.jwt_expiration =
-          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
-      c->cached.service_url = gpr_strdup(context.service_url);
-      c->cached.jwt_md = grpc_credentials_md_store_create(1);
-      grpc_credentials_md_store_add_cstrings(
-          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
-      gpr_free(md_value);
-      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
-    }
-    gpr_mu_unlock(&c->cache_mu);
-  }
-
-  if (jwt_md != NULL) {
-    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
-       GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(jwt_md);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  }
-}
-
-static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
-                                                  jwt_get_request_metadata};
-
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime) {
-  grpc_service_account_jwt_access_credentials *c;
-  if (!grpc_auth_json_key_is_valid(&key)) {
-    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
-  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->base.vtable = &jwt_vtable;
-  c->key = key;
-  c->jwt_lifetime = token_lifetime;
-  gpr_mu_init(&c->cache_mu);
-  jwt_reset_cache(c);
-  return &c->base;
-}
-
-grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
-    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_service_account_jwt_access_credentials_create("
-      "json_key=%s, "
-      "token_lifetime="
-      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
-      "reserved=%p)",
-      5,
-      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
-       (int)token_lifetime.clock_type, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
-}
-
-/* -- Oauth2TokenFetcher credentials -- */
-
-static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-  gpr_mu_destroy(&c->mu);
-  grpc_httpcli_context_destroy(&c->httpcli_context);
-}
-
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const grpc_http_response *response, grpc_credentials_md_store **token_md,
-    gpr_timespec *token_lifetime) {
-  char *null_terminated_body = NULL;
-  char *new_access_token = NULL;
-  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
-  grpc_json *json = NULL;
-
-  if (response == NULL) {
-    gpr_log(GPR_ERROR, "Received NULL response.");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  }
-
-  if (response->body_length > 0) {
-    null_terminated_body = gpr_malloc(response->body_length + 1);
-    null_terminated_body[response->body_length] = '\0';
-    memcpy(null_terminated_body, response->body, response->body_length);
-  }
-
-  if (response->status != 200) {
-    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
-            response->status,
-            null_terminated_body != NULL ? null_terminated_body : "");
-    status = GRPC_CREDENTIALS_ERROR;
-    goto end;
-  } else {
-    grpc_json *access_token = NULL;
-    grpc_json *token_type = NULL;
-    grpc_json *expires_in = NULL;
-    grpc_json *ptr;
-    json = grpc_json_parse_string(null_terminated_body);
-    if (json == NULL) {
-      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (json->type != GRPC_JSON_OBJECT) {
-      gpr_log(GPR_ERROR, "Response should be a JSON object");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    for (ptr = json->child; ptr; ptr = ptr->next) {
-      if (strcmp(ptr->key, "access_token") == 0) {
-        access_token = ptr;
-      } else if (strcmp(ptr->key, "token_type") == 0) {
-        token_type = ptr;
-      } else if (strcmp(ptr->key, "expires_in") == 0) {
-        expires_in = ptr;
-      }
-    }
-    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
-      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
-      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
-      status = GRPC_CREDENTIALS_ERROR;
-      goto end;
-    }
-    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
-                 access_token->value);
-    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
-    token_lifetime->tv_nsec = 0;
-    token_lifetime->clock_type = GPR_TIMESPAN;
-    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
-    *token_md = grpc_credentials_md_store_create(1);
-    grpc_credentials_md_store_add_cstrings(
-        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
-    status = GRPC_CREDENTIALS_OK;
-  }
-
-end:
-  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
-    grpc_credentials_md_store_unref(*token_md);
-    *token_md = NULL;
-  }
-  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
-  if (new_access_token != NULL) gpr_free(new_access_token);
-  if (json != NULL) grpc_json_destroy(json);
-  return status;
-}
-
-static void on_oauth2_token_fetcher_http_response(
-    grpc_exec_ctx *exec_ctx, void *user_data,
-    const grpc_http_response *response) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)r->creds;
-  gpr_timespec token_lifetime;
-  grpc_credentials_status status;
-
-  gpr_mu_lock(&c->mu);
-  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
-      response, &c->access_token_md, &token_lifetime);
-  if (status == GRPC_CREDENTIALS_OK) {
-    c->token_expiration =
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
-    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
-          c->access_token_md->num_entries, status);
-  } else {
-    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-    r->cb(exec_ctx, r->user_data, NULL, 0, status);
-  }
-  gpr_mu_unlock(&c->mu);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void oauth2_token_fetcher_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      (grpc_oauth2_token_fetcher_credentials *)creds;
-  gpr_timespec refresh_threshold = gpr_time_from_seconds(
-      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
-  grpc_credentials_md_store *cached_access_token_md = NULL;
-  {
-    gpr_mu_lock(&c->mu);
-    if (c->access_token_md != NULL &&
-        (gpr_time_cmp(
-             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
-             refresh_threshold) > 0)) {
-      cached_access_token_md =
-          grpc_credentials_md_store_ref(c->access_token_md);
-    }
-    gpr_mu_unlock(&c->mu);
-  }
-  if (cached_access_token_md != NULL) {
-    cb(exec_ctx, user_data, cached_access_token_md->entries,
-       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
-    grpc_credentials_md_store_unref(cached_access_token_md);
-  } else {
-    c->fetch_func(
-        exec_ctx,
-        grpc_credentials_metadata_request_create(creds, cb, user_data),
-        &c->httpcli_context, pollset, on_oauth2_token_fetcher_http_response,
-        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
-  }
-}
-
-static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
-                                      grpc_fetch_oauth2_func fetch_func) {
-  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  gpr_ref_init(&c->base.refcount, 1);
-  gpr_mu_init(&c->mu);
-  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
-  c->fetch_func = fetch_func;
-  grpc_httpcli_context_init(&c->httpcli_context);
-}
-
-/* -- GoogleComputeEngine credentials. -- */
-
-static grpc_call_credentials_vtable compute_engine_vtable = {
-    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void compute_engine_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_http_header header = {"Metadata-Flavor", "Google"};
-  grpc_httpcli_request request;
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
-  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline,
-                   response_cb, metadata_req);
-}
-
-grpc_call_credentials *grpc_google_compute_engine_credentials_create(
-    void *reserved) {
-  grpc_oauth2_token_fetcher_credentials *c =
-      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
-  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
-  c->base.vtable = &compute_engine_vtable;
-  return &c->base;
-}
-
-/* -- GoogleRefreshToken credentials. -- */
-
-static void refresh_token_destruct(grpc_call_credentials *creds) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)creds;
-  grpc_auth_refresh_token_destruct(&c->refresh_token);
-  oauth2_token_fetcher_destruct(&c->base.base);
-}
-
-static grpc_call_credentials_vtable refresh_token_vtable = {
-    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
-
-static void refresh_token_fetch_oauth2(
-    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
-    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
-    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
-  grpc_google_refresh_token_credentials *c =
-      (grpc_google_refresh_token_credentials *)metadata_req->creds;
-  grpc_http_header header = {"Content-Type",
-                             "application/x-www-form-urlencoded"};
-  grpc_httpcli_request request;
-  char *body = NULL;
-  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
-               c->refresh_token.client_id, c->refresh_token.client_secret,
-               c->refresh_token.refresh_token);
-  memset(&request, 0, sizeof(grpc_httpcli_request));
-  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
-  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
-  request.http.hdr_count = 1;
-  request.http.hdrs = &header;
-  request.handshaker = &grpc_httpcli_ssl;
-  grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body,
-                    strlen(body), deadline, response_cb, metadata_req);
-  gpr_free(body);
-}
-
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token refresh_token) {
-  grpc_google_refresh_token_credentials *c;
-  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
-    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
-    return NULL;
-  }
-  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
-  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
-  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
-  c->base.base.vtable = &refresh_token_vtable;
-  c->refresh_token = refresh_token;
-  return &c->base.base;
-}
-
-grpc_call_credentials *grpc_google_refresh_token_credentials_create(
-    const char *json_refresh_token, void *reserved) {
-  GRPC_API_TRACE(
-      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
-      "reserved=%p)",
-      2, (json_refresh_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
-      grpc_auth_refresh_token_create_from_string(json_refresh_token));
-}
-
-/* -- Metadata-only credentials. -- */
-
-static void md_only_test_destruct(grpc_call_credentials *creds) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-  grpc_credentials_md_store_unref(c->md_store);
-}
-
-static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
-                                          void *user_data, bool success) {
-  grpc_credentials_metadata_request *r =
-      (grpc_credentials_metadata_request *)user_data;
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
-  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
-        GRPC_CREDENTIALS_OK);
-  grpc_credentials_metadata_request_destroy(r);
-}
-
-static void md_only_test_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
-
-  if (c->is_async) {
-    grpc_credentials_metadata_request *cb_arg =
-        grpc_credentials_metadata_request_create(creds, cb, user_data);
-    grpc_executor_enqueue(
-        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
-  } else {
-    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable md_only_test_vtable = {
-    md_only_test_destruct, md_only_test_get_request_metadata};
-
-grpc_call_credentials *grpc_md_only_test_credentials_create(
-    const char *md_key, const char *md_value, int is_async) {
-  grpc_md_only_test_credentials *c =
-      gpr_malloc(sizeof(grpc_md_only_test_credentials));
-  memset(c, 0, sizeof(grpc_md_only_test_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &md_only_test_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->md_store = grpc_credentials_md_store_create(1);
-  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
-  c->is_async = is_async;
-  return &c->base;
-}
-
-/* -- Oauth2 Access Token credentials. -- */
-
-static void access_token_destruct(grpc_call_credentials *creds) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  grpc_credentials_md_store_unref(c->access_token_md);
-}
-
-static void access_token_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
-  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable access_token_vtable = {
-    access_token_destruct, access_token_get_request_metadata};
-
-grpc_call_credentials *grpc_access_token_credentials_create(
-    const char *access_token, void *reserved) {
-  grpc_access_token_credentials *c =
-      gpr_malloc(sizeof(grpc_access_token_credentials));
-  char *token_md_value;
-  GRPC_API_TRACE(
-      "grpc_access_token_credentials_create(access_token=%s, "
-      "reserved=%p)",
-      2, (access_token, reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(grpc_access_token_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
-  c->base.vtable = &access_token_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->access_token_md = grpc_credentials_md_store_create(1);
-  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
-  grpc_credentials_md_store_add_cstrings(
-      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
-  gpr_free(token_md_value);
-  return &c->base;
-}
-
-/* -- Fake transport security credentials. -- */
-
-static grpc_security_status fake_transport_security_create_security_connector(
-    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  *sc = grpc_fake_channel_security_connector_create(call_creds);
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_security_status
-fake_transport_security_server_create_security_connector(
-    grpc_server_credentials *c, grpc_server_security_connector **sc) {
-  *sc = grpc_fake_server_security_connector_create();
-  return GRPC_SECURITY_OK;
-}
-
-static grpc_channel_credentials_vtable
-    fake_transport_security_credentials_vtable = {
-        NULL, fake_transport_security_create_security_connector};
-
-static grpc_server_credentials_vtable
-    fake_transport_security_server_credentials_vtable = {
-        NULL, fake_transport_security_server_create_security_connector};
-
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
-    void) {
-  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
-  memset(c, 0, sizeof(grpc_channel_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  c->vtable = &fake_transport_security_credentials_vtable;
-  gpr_ref_init(&c->refcount, 1);
-  return c;
-}
-
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void) {
-  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
-  memset(c, 0, sizeof(grpc_server_credentials));
-  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
-  gpr_ref_init(&c->refcount, 1);
-  c->vtable = &fake_transport_security_server_credentials_vtable;
-  return c;
-}
-
-/* -- Composite call credentials. -- */
-
-typedef struct {
-  grpc_composite_call_credentials *composite_creds;
-  size_t creds_index;
-  grpc_credentials_md_store *md_elems;
-  grpc_auth_metadata_context auth_md_context;
-  void *user_data;
-  grpc_pollset *pollset;
-  grpc_credentials_metadata_cb cb;
-} grpc_composite_call_credentials_metadata_context;
-
-static void composite_call_destruct(grpc_call_credentials *creds) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  size_t i;
-  for (i = 0; i < c->inner.num_creds; i++) {
-    grpc_call_credentials_unref(c->inner.creds_array[i]);
-  }
-  gpr_free(c->inner.creds_array);
-}
-
-static void composite_call_md_context_destroy(
-    grpc_composite_call_credentials_metadata_context *ctx) {
-  grpc_credentials_md_store_unref(ctx->md_elems);
-  gpr_free(ctx);
-}
-
-static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
-                                       grpc_credentials_md *md_elems,
-                                       size_t num_md,
-                                       grpc_credentials_status status) {
-  grpc_composite_call_credentials_metadata_context *ctx =
-      (grpc_composite_call_credentials_metadata_context *)user_data;
-  if (status != GRPC_CREDENTIALS_OK) {
-    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
-    return;
-  }
-
-  /* Copy the metadata in the context. */
-  if (num_md > 0) {
-    size_t i;
-    for (i = 0; i < num_md; i++) {
-      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
-                                    md_elems[i].value);
-    }
-  }
-
-  /* See if we need to get some more metadata. */
-  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
-    grpc_call_credentials *inner_creds =
-        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
-    grpc_call_credentials_get_request_metadata(
-        exec_ctx, inner_creds, ctx->pollset, ctx->auth_md_context,
-        composite_call_metadata_cb, ctx);
-    return;
-  }
-
-  /* We're done!. */
-  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
-          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
-  composite_call_md_context_destroy(ctx);
-}
-
-static void composite_call_get_request_metadata(
-    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
-    grpc_pollset *pollset, grpc_auth_metadata_context auth_md_context,
-    grpc_credentials_metadata_cb cb, void *user_data) {
-  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
-  grpc_composite_call_credentials_metadata_context *ctx;
-
-  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
-  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
-  ctx->auth_md_context = auth_md_context;
-  ctx->user_data = user_data;
-  ctx->cb = cb;
-  ctx->composite_creds = c;
-  ctx->pollset = pollset;
-  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
-  grpc_call_credentials_get_request_metadata(
-      exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset,
-      auth_md_context, composite_call_metadata_cb, ctx);
-}
-
-static grpc_call_credentials_vtable composite_call_credentials_vtable = {
-    composite_call_destruct, composite_call_get_request_metadata};
-
-static grpc_call_credentials_array get_creds_array(
-    grpc_call_credentials **creds_addr) {
-  grpc_call_credentials_array result;
-  grpc_call_credentials *creds = *creds_addr;
-  result.creds_array = creds_addr;
-  result.num_creds = 1;
-  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    result = *grpc_composite_call_credentials_get_credentials(creds);
-  }
-  return result;
-}
-
-grpc_call_credentials *grpc_composite_call_credentials_create(
-    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
-    void *reserved) {
-  size_t i;
-  size_t creds_array_byte_size;
-  grpc_call_credentials_array creds1_array;
-  grpc_call_credentials_array creds2_array;
-  grpc_composite_call_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
-      "reserved=%p)",
-      3, (creds1, creds2, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(creds1 != NULL);
-  GPR_ASSERT(creds2 != NULL);
-  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
-  memset(c, 0, sizeof(grpc_composite_call_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
-  c->base.vtable = &composite_call_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  creds1_array = get_creds_array(&creds1);
-  creds2_array = get_creds_array(&creds2);
-  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
-  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
-  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
-  memset(c->inner.creds_array, 0, creds_array_byte_size);
-  for (i = 0; i < creds1_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
-    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
-  }
-  for (i = 0; i < creds2_array.num_creds; i++) {
-    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
-    c->inner.creds_array[i + creds1_array.num_creds] =
-        grpc_call_credentials_ref(cur_creds);
-  }
-  return &c->base;
-}
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
-  const grpc_composite_call_credentials *c =
-      (const grpc_composite_call_credentials *)creds;
-  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
-  return &c->inner;
-}
-
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds) {
-  size_t i;
-  if (strcmp(creds->type, type) == 0) {
-    if (composite_creds != NULL) *composite_creds = NULL;
-    return creds;
-  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
-    const grpc_call_credentials_array *inner_creds_array =
-        grpc_composite_call_credentials_get_credentials(creds);
-    for (i = 0; i < inner_creds_array->num_creds; i++) {
-      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
-        if (composite_creds != NULL) *composite_creds = creds;
-        return inner_creds_array->creds_array[i];
-      }
-    }
-  }
-  return NULL;
-}
-
-/* -- IAM credentials. -- */
-
-static void iam_destruct(grpc_call_credentials *creds) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  grpc_credentials_md_store_unref(c->iam_md);
-}
-
-static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                     grpc_call_credentials *creds,
-                                     grpc_pollset *pollset,
-                                     grpc_auth_metadata_context context,
-                                     grpc_credentials_metadata_cb cb,
-                                     void *user_data) {
-  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
-  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
-     GRPC_CREDENTIALS_OK);
-}
-
-static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
-                                                  iam_get_request_metadata};
-
-grpc_call_credentials *grpc_google_iam_credentials_create(
-    const char *token, const char *authority_selector, void *reserved) {
-  grpc_google_iam_credentials *c;
-  GRPC_API_TRACE(
-      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
-      "reserved=%p)",
-      3, (token, authority_selector, reserved));
-  GPR_ASSERT(reserved == NULL);
-  GPR_ASSERT(token != NULL);
-  GPR_ASSERT(authority_selector != NULL);
-  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
-  memset(c, 0, sizeof(grpc_google_iam_credentials));
-  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
-  c->base.vtable = &iam_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->iam_md = grpc_credentials_md_store_create(2);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
-  grpc_credentials_md_store_add_cstrings(
-      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
-  return &c->base;
-}
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  void *user_data;
-  grpc_credentials_metadata_cb cb;
-} grpc_metadata_plugin_request;
-
-static void plugin_destruct(grpc_call_credentials *creds) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
-    c->plugin.destroy(c->plugin.state);
-  }
-}
-
-static void plugin_md_request_metadata_ready(void *request,
-                                             const grpc_metadata *md,
-                                             size_t num_md,
-                                             grpc_status_code status,
-                                             const char *error_details) {
-  /* called from application code */
-  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
-  if (status != GRPC_STATUS_OK) {
-    if (error_details != NULL) {
-      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
-              error_details);
-    }
-    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
-  } else {
-    size_t i;
-    grpc_credentials_md *md_array = NULL;
-    if (num_md > 0) {
-      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
-      for (i = 0; i < num_md; i++) {
-        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
-        md_array[i].value =
-            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
-      }
-    }
-    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
-    if (md_array != NULL) {
-      for (i = 0; i < num_md; i++) {
-        gpr_slice_unref(md_array[i].key);
-        gpr_slice_unref(md_array[i].value);
-      }
-      gpr_free(md_array);
-    }
-  }
-  gpr_free(r);
-  grpc_exec_ctx_finish(&exec_ctx);
-}
-
-static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
-                                        grpc_call_credentials *creds,
-                                        grpc_pollset *pollset,
-                                        grpc_auth_metadata_context context,
-                                        grpc_credentials_metadata_cb cb,
-                                        void *user_data) {
-  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
-  if (c->plugin.get_metadata != NULL) {
-    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
-    memset(request, 0, sizeof(*request));
-    request->user_data = user_data;
-    request->cb = cb;
-    c->plugin.get_metadata(c->plugin.state, context,
-                           plugin_md_request_metadata_ready, request);
-  } else {
-    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
-  }
-}
-
-static grpc_call_credentials_vtable plugin_vtable = {
-    plugin_destruct, plugin_get_request_metadata};
-
-grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
-    grpc_metadata_credentials_plugin plugin, void *reserved) {
-  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
-  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
-                 (reserved));
-  GPR_ASSERT(reserved == NULL);
-  memset(c, 0, sizeof(*c));
-  c->base.type = plugin.type;
-  c->base.vtable = &plugin_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->plugin = plugin;
-  return &c->base;
-}
-
-/* -- Composite channel credentials. -- */
-
-static void composite_channel_destruct(grpc_channel_credentials *creds) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_channel_credentials_unref(c->inner_creds);
-  grpc_call_credentials_unref(c->call_creds);
-}
-
-static grpc_security_status composite_channel_create_security_connector(
-    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
-    const char *target, const grpc_channel_args *args,
-    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
-  grpc_composite_channel_credentials *c =
-      (grpc_composite_channel_credentials *)creds;
-  grpc_security_status status = GRPC_SECURITY_ERROR;
-
-  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
-             c->inner_creds->vtable != NULL &&
-             c->inner_creds->vtable->create_security_connector != NULL);
-  /* If we are passed a call_creds, create a call composite to pass it
-     downstream. */
-  if (call_creds != NULL) {
-    grpc_call_credentials *composite_call_creds =
-        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, composite_call_creds, target, args, sc, new_args);
-    grpc_call_credentials_unref(composite_call_creds);
-  } else {
-    status = c->inner_creds->vtable->create_security_connector(
-        c->inner_creds, c->call_creds, target, args, sc, new_args);
-  }
-  return status;
-}
-
-static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
-    composite_channel_destruct, composite_channel_create_security_connector};
-
-grpc_channel_credentials *grpc_composite_channel_credentials_create(
-    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
-    void *reserved) {
-  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
-  memset(c, 0, sizeof(*c));
-  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
-  GRPC_API_TRACE(
-      "grpc_composite_channel_credentials_create(channel_creds=%p, "
-      "call_creds=%p, reserved=%p)",
-      3, (channel_creds, call_creds, reserved));
-  c->base.type = channel_creds->type;
-  c->base.vtable = &composite_channel_credentials_vtable;
-  gpr_ref_init(&c->base.refcount, 1);
-  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
-  c->call_creds = grpc_call_credentials_ref(call_creds);
-  return &c->base;
-}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.c b/src/core/lib/security/credentials/composite/composite_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..18189a8fb8c75cb3636d5ed26faee870dafc3dbf
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.c
@@ -0,0 +1,262 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Composite call credentials. -- */
+
+typedef struct {
+  grpc_composite_call_credentials *composite_creds;
+  size_t creds_index;
+  grpc_credentials_md_store *md_elems;
+  grpc_auth_metadata_context auth_md_context;
+  void *user_data;
+  grpc_pollset *pollset;
+  grpc_credentials_metadata_cb cb;
+} grpc_composite_call_credentials_metadata_context;
+
+static void composite_call_destruct(grpc_call_credentials *creds) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->inner.num_creds; i++) {
+    grpc_call_credentials_unref(c->inner.creds_array[i]);
+  }
+  gpr_free(c->inner.creds_array);
+}
+
+static void composite_call_md_context_destroy(
+    grpc_composite_call_credentials_metadata_context *ctx) {
+  grpc_credentials_md_store_unref(ctx->md_elems);
+  gpr_free(ctx);
+}
+
+static void composite_call_metadata_cb(grpc_exec_ctx *exec_ctx, void *user_data,
+                                       grpc_credentials_md *md_elems,
+                                       size_t num_md,
+                                       grpc_credentials_status status) {
+  grpc_composite_call_credentials_metadata_context *ctx =
+      (grpc_composite_call_credentials_metadata_context *)user_data;
+  if (status != GRPC_CREDENTIALS_OK) {
+    ctx->cb(exec_ctx, ctx->user_data, NULL, 0, status);
+    return;
+  }
+
+  /* Copy the metadata in the context. */
+  if (num_md > 0) {
+    size_t i;
+    for (i = 0; i < num_md; i++) {
+      grpc_credentials_md_store_add(ctx->md_elems, md_elems[i].key,
+                                    md_elems[i].value);
+    }
+  }
+
+  /* See if we need to get some more metadata. */
+  if (ctx->creds_index < ctx->composite_creds->inner.num_creds) {
+    grpc_call_credentials *inner_creds =
+        ctx->composite_creds->inner.creds_array[ctx->creds_index++];
+    grpc_call_credentials_get_request_metadata(
+        exec_ctx, inner_creds, ctx->pollset, ctx->auth_md_context,
+        composite_call_metadata_cb, ctx);
+    return;
+  }
+
+  /* We're done!. */
+  ctx->cb(exec_ctx, ctx->user_data, ctx->md_elems->entries,
+          ctx->md_elems->num_entries, GRPC_CREDENTIALS_OK);
+  composite_call_md_context_destroy(ctx);
+}
+
+static void composite_call_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pollset *pollset, grpc_auth_metadata_context auth_md_context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_composite_call_credentials *c = (grpc_composite_call_credentials *)creds;
+  grpc_composite_call_credentials_metadata_context *ctx;
+
+  ctx = gpr_malloc(sizeof(grpc_composite_call_credentials_metadata_context));
+  memset(ctx, 0, sizeof(grpc_composite_call_credentials_metadata_context));
+  ctx->auth_md_context = auth_md_context;
+  ctx->user_data = user_data;
+  ctx->cb = cb;
+  ctx->composite_creds = c;
+  ctx->pollset = pollset;
+  ctx->md_elems = grpc_credentials_md_store_create(c->inner.num_creds);
+  grpc_call_credentials_get_request_metadata(
+      exec_ctx, c->inner.creds_array[ctx->creds_index++], pollset,
+      auth_md_context, composite_call_metadata_cb, ctx);
+}
+
+static grpc_call_credentials_vtable composite_call_credentials_vtable = {
+    composite_call_destruct, composite_call_get_request_metadata};
+
+static grpc_call_credentials_array get_creds_array(
+    grpc_call_credentials **creds_addr) {
+  grpc_call_credentials_array result;
+  grpc_call_credentials *creds = *creds_addr;
+  result.creds_array = creds_addr;
+  result.num_creds = 1;
+  if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    result = *grpc_composite_call_credentials_get_credentials(creds);
+  }
+  return result;
+}
+
+grpc_call_credentials *grpc_composite_call_credentials_create(
+    grpc_call_credentials *creds1, grpc_call_credentials *creds2,
+    void *reserved) {
+  size_t i;
+  size_t creds_array_byte_size;
+  grpc_call_credentials_array creds1_array;
+  grpc_call_credentials_array creds2_array;
+  grpc_composite_call_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_composite_call_credentials_create(creds1=%p, creds2=%p, "
+      "reserved=%p)",
+      3, (creds1, creds2, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(creds1 != NULL);
+  GPR_ASSERT(creds2 != NULL);
+  c = gpr_malloc(sizeof(grpc_composite_call_credentials));
+  memset(c, 0, sizeof(grpc_composite_call_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE;
+  c->base.vtable = &composite_call_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  creds1_array = get_creds_array(&creds1);
+  creds2_array = get_creds_array(&creds2);
+  c->inner.num_creds = creds1_array.num_creds + creds2_array.num_creds;
+  creds_array_byte_size = c->inner.num_creds * sizeof(grpc_call_credentials *);
+  c->inner.creds_array = gpr_malloc(creds_array_byte_size);
+  memset(c->inner.creds_array, 0, creds_array_byte_size);
+  for (i = 0; i < creds1_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds1_array.creds_array[i];
+    c->inner.creds_array[i] = grpc_call_credentials_ref(cur_creds);
+  }
+  for (i = 0; i < creds2_array.num_creds; i++) {
+    grpc_call_credentials *cur_creds = creds2_array.creds_array[i];
+    c->inner.creds_array[i + creds1_array.num_creds] =
+        grpc_call_credentials_ref(cur_creds);
+  }
+  return &c->base;
+}
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(grpc_call_credentials *creds) {
+  const grpc_composite_call_credentials *c =
+      (const grpc_composite_call_credentials *)creds;
+  GPR_ASSERT(strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0);
+  return &c->inner;
+}
+
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds) {
+  size_t i;
+  if (strcmp(creds->type, type) == 0) {
+    if (composite_creds != NULL) *composite_creds = NULL;
+    return creds;
+  } else if (strcmp(creds->type, GRPC_CALL_CREDENTIALS_TYPE_COMPOSITE) == 0) {
+    const grpc_call_credentials_array *inner_creds_array =
+        grpc_composite_call_credentials_get_credentials(creds);
+    for (i = 0; i < inner_creds_array->num_creds; i++) {
+      if (strcmp(type, inner_creds_array->creds_array[i]->type) == 0) {
+        if (composite_creds != NULL) *composite_creds = creds;
+        return inner_creds_array->creds_array[i];
+      }
+    }
+  }
+  return NULL;
+}
+
+/* -- Composite channel credentials. -- */
+
+static void composite_channel_destruct(grpc_channel_credentials *creds) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_channel_credentials_unref(c->inner_creds);
+  grpc_call_credentials_unref(c->call_creds);
+}
+
+static grpc_security_status composite_channel_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_composite_channel_credentials *c =
+      (grpc_composite_channel_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_ERROR;
+
+  GPR_ASSERT(c->inner_creds != NULL && c->call_creds != NULL &&
+             c->inner_creds->vtable != NULL &&
+             c->inner_creds->vtable->create_security_connector != NULL);
+  /* If we are passed a call_creds, create a call composite to pass it
+     downstream. */
+  if (call_creds != NULL) {
+    grpc_call_credentials *composite_call_creds =
+        grpc_composite_call_credentials_create(c->call_creds, call_creds, NULL);
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, composite_call_creds, target, args, sc, new_args);
+    grpc_call_credentials_unref(composite_call_creds);
+  } else {
+    status = c->inner_creds->vtable->create_security_connector(
+        c->inner_creds, c->call_creds, target, args, sc, new_args);
+  }
+  return status;
+}
+
+static grpc_channel_credentials_vtable composite_channel_credentials_vtable = {
+    composite_channel_destruct, composite_channel_create_security_connector};
+
+grpc_channel_credentials *grpc_composite_channel_credentials_create(
+    grpc_channel_credentials *channel_creds, grpc_call_credentials *call_creds,
+    void *reserved) {
+  grpc_composite_channel_credentials *c = gpr_malloc(sizeof(*c));
+  memset(c, 0, sizeof(*c));
+  GPR_ASSERT(channel_creds != NULL && call_creds != NULL && reserved == NULL);
+  GRPC_API_TRACE(
+      "grpc_composite_channel_credentials_create(channel_creds=%p, "
+      "call_creds=%p, reserved=%p)",
+      3, (channel_creds, call_creds, reserved));
+  c->base.type = channel_creds->type;
+  c->base.vtable = &composite_channel_credentials_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->inner_creds = grpc_channel_credentials_ref(channel_creds);
+  c->call_creds = grpc_call_credentials_ref(call_creds);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/composite/composite_credentials.h b/src/core/lib/security/credentials/composite/composite_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..0d8966f464d331c795f0e31eab4f3916b5a59bc9
--- /dev/null
+++ b/src/core/lib/security/credentials/composite/composite_credentials.h
@@ -0,0 +1,72 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials **creds_array;
+  size_t num_creds;
+} grpc_call_credentials_array;
+
+const grpc_call_credentials_array *
+grpc_composite_call_credentials_get_credentials(
+    grpc_call_credentials *composite_creds);
+
+/* Returns creds if creds is of the specified type or the inner creds of the
+   specified type (if found), if the creds is of type COMPOSITE.
+   If composite_creds is not NULL, *composite_creds will point to creds if of
+   type COMPOSITE in case of success. */
+grpc_call_credentials *grpc_credentials_contains_type(
+    grpc_call_credentials *creds, const char *type,
+    grpc_call_credentials **composite_creds);
+
+/* -- Channel composite credentials. -- */
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_channel_credentials *inner_creds;
+  grpc_call_credentials *call_creds;
+} grpc_composite_channel_credentials;
+
+/* -- Composite credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_call_credentials_array inner;
+} grpc_composite_call_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_COMPOSITE_COMPOSITE_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/credentials.c b/src/core/lib/security/credentials/credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..3dde6e587ded2837ed97b1c5638d0e5bef527ae9
--- /dev/null
+++ b/src/core/lib/security/credentials/credentials.c
@@ -0,0 +1,231 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/http/httpcli.h"
+#include "src/core/lib/http/parser.h"
+#include "src/core/lib/iomgr/executor.h"
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/support/string.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+#include <grpc/support/time.h>
+
+/* -- Common. -- */
+
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data) {
+  grpc_credentials_metadata_request *r =
+      gpr_malloc(sizeof(grpc_credentials_metadata_request));
+  r->creds = grpc_call_credentials_ref(creds);
+  r->cb = cb;
+  r->user_data = user_data;
+  return r;
+}
+
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r) {
+  grpc_call_credentials_unref(r->creds);
+  gpr_free(r);
+}
+
+grpc_channel_credentials *grpc_channel_credentials_ref(
+    grpc_channel_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_channel_credentials_unref(grpc_channel_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_channel_credentials_release(grpc_channel_credentials *creds) {
+  GRPC_API_TRACE("grpc_channel_credentials_release(creds=%p)", 1, (creds));
+  grpc_channel_credentials_unref(creds);
+}
+
+grpc_call_credentials *grpc_call_credentials_ref(grpc_call_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_call_credentials_unref(grpc_call_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    gpr_free(creds);
+  }
+}
+
+void grpc_call_credentials_release(grpc_call_credentials *creds) {
+  GRPC_API_TRACE("grpc_call_credentials_release(creds=%p)", 1, (creds));
+  grpc_call_credentials_unref(creds);
+}
+
+void grpc_call_credentials_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  if (creds == NULL || creds->vtable->get_request_metadata == NULL) {
+    if (cb != NULL) {
+      cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+    }
+    return;
+  }
+  creds->vtable->get_request_metadata(exec_ctx, creds, pollset, context, cb,
+                                      user_data);
+}
+
+grpc_security_status grpc_channel_credentials_create_security_connector(
+    grpc_channel_credentials *channel_creds, const char *target,
+    const grpc_channel_args *args, grpc_channel_security_connector **sc,
+    grpc_channel_args **new_args) {
+  *new_args = NULL;
+  if (channel_creds == NULL) {
+    return GRPC_SECURITY_ERROR;
+  }
+  GPR_ASSERT(channel_creds->vtable->create_security_connector != NULL);
+  return channel_creds->vtable->create_security_connector(
+      channel_creds, NULL, target, args, sc, new_args);
+}
+
+grpc_server_credentials *grpc_server_credentials_ref(
+    grpc_server_credentials *creds) {
+  if (creds == NULL) return NULL;
+  gpr_ref(&creds->refcount);
+  return creds;
+}
+
+void grpc_server_credentials_unref(grpc_server_credentials *creds) {
+  if (creds == NULL) return;
+  if (gpr_unref(&creds->refcount)) {
+    if (creds->vtable->destruct != NULL) creds->vtable->destruct(creds);
+    if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+      creds->processor.destroy(creds->processor.state);
+    }
+    gpr_free(creds);
+  }
+}
+
+void grpc_server_credentials_release(grpc_server_credentials *creds) {
+  GRPC_API_TRACE("grpc_server_credentials_release(creds=%p)", 1, (creds));
+  grpc_server_credentials_unref(creds);
+}
+
+grpc_security_status grpc_server_credentials_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  if (creds == NULL || creds->vtable->create_security_connector == NULL) {
+    gpr_log(GPR_ERROR, "Server credentials cannot create security context.");
+    return GRPC_SECURITY_ERROR;
+  }
+  return creds->vtable->create_security_connector(creds, sc);
+}
+
+void grpc_server_credentials_set_auth_metadata_processor(
+    grpc_server_credentials *creds, grpc_auth_metadata_processor processor) {
+  GRPC_API_TRACE(
+      "grpc_server_credentials_set_auth_metadata_processor("
+      "creds=%p, "
+      "processor=grpc_auth_metadata_processor { process: %p, state: %p })",
+      3, (creds, (void *)(intptr_t)processor.process, processor.state));
+  if (creds == NULL) return;
+  if (creds->processor.destroy != NULL && creds->processor.state != NULL) {
+    creds->processor.destroy(creds->processor.state);
+  }
+  creds->processor = processor;
+}
+
+static void server_credentials_pointer_arg_destroy(void *p) {
+  grpc_server_credentials_unref(p);
+}
+
+static void *server_credentials_pointer_arg_copy(void *p) {
+  return grpc_server_credentials_ref(p);
+}
+
+static int server_credentials_pointer_cmp(void *a, void *b) {
+  return GPR_ICMP(a, b);
+}
+
+static const grpc_arg_pointer_vtable cred_ptr_vtable = {
+    server_credentials_pointer_arg_copy, server_credentials_pointer_arg_destroy,
+    server_credentials_pointer_cmp};
+
+grpc_arg grpc_server_credentials_to_arg(grpc_server_credentials *p) {
+  grpc_arg arg;
+  memset(&arg, 0, sizeof(grpc_arg));
+  arg.type = GRPC_ARG_POINTER;
+  arg.key = GRPC_SERVER_CREDENTIALS_ARG;
+  arg.value.pointer.p = p;
+  arg.value.pointer.vtable = &cred_ptr_vtable;
+  return arg;
+}
+
+grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg) {
+  if (strcmp(arg->key, GRPC_SERVER_CREDENTIALS_ARG) != 0) return NULL;
+  if (arg->type != GRPC_ARG_POINTER) {
+    gpr_log(GPR_ERROR, "Invalid type %d for arg %s", arg->type,
+            GRPC_SERVER_CREDENTIALS_ARG);
+    return NULL;
+  }
+  return arg->value.pointer.p;
+}
+
+grpc_server_credentials *grpc_find_server_credentials_in_args(
+    const grpc_channel_args *args) {
+  size_t i;
+  if (args == NULL) return NULL;
+  for (i = 0; i < args->num_args; i++) {
+    grpc_server_credentials *p =
+        grpc_server_credentials_from_arg(&args->args[i]);
+    if (p != NULL) return p;
+  }
+  return NULL;
+}
diff --git a/src/core/lib/security/credentials.h b/src/core/lib/security/credentials/credentials.h
similarity index 62%
rename from src/core/lib/security/credentials.h
rename to src/core/lib/security/credentials/credentials.h
index 0373ceaa3fc5c3157fe752967e9d1797bd0b787e..5f44c7c3e30d22cb867318904a7b449c364d8e7c 100644
--- a/src/core/lib/security/credentials.h
+++ b/src/core/lib/security/credentials/credentials.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
-#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H
 
 #include <grpc/grpc.h>
 #include <grpc/grpc_security.h>
@@ -41,8 +41,7 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
-#include "src/core/lib/security/json_token.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 struct grpc_http_response;
 
@@ -69,10 +68,6 @@ typedef enum {
   "x-goog-iam-authorization-token"
 #define GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY "x-goog-iam-authority-selector"
 
-#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
-#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
-  "application_default_credentials.json"
-
 #define GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS 60
 
 #define GRPC_COMPUTE_ENGINE_METADATA_HOST "metadata"
@@ -188,48 +183,11 @@ void grpc_call_credentials_get_request_metadata(
     grpc_pollset *pollset, grpc_auth_metadata_context context,
     grpc_credentials_metadata_cb cb, void *user_data);
 
-typedef struct {
-  grpc_call_credentials **creds_array;
-  size_t num_creds;
-} grpc_call_credentials_array;
-
-const grpc_call_credentials_array *
-grpc_composite_call_credentials_get_credentials(
-    grpc_call_credentials *composite_creds);
-
-/* Returns creds if creds is of the specified type or the inner creds of the
-   specified type (if found), if the creds is of type COMPOSITE.
-   If composite_creds is not NULL, *composite_creds will point to creds if of
-   type COMPOSITE in case of success. */
-grpc_call_credentials *grpc_credentials_contains_type(
-    grpc_call_credentials *creds, const char *type,
-    grpc_call_credentials **composite_creds);
-
-/* Exposed for testing only. */
-grpc_credentials_status
-grpc_oauth2_token_fetcher_credentials_parse_server_response(
-    const struct grpc_http_response *response,
-    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
-
-void grpc_flush_cached_google_default_credentials(void);
-
 /* Metadata-only credentials with the specified key and value where
    asynchronicity can be simulated for testing. */
 grpc_call_credentials *grpc_md_only_test_credentials_create(
     const char *md_key, const char *md_value, int is_async);
 
-/* Private constructor for jwt credentials from an already parsed json key.
-   Takes ownership of the key. */
-grpc_call_credentials *
-grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
-    grpc_auth_json_key key, gpr_timespec token_lifetime);
-
-/* Private constructor for refresh token credentials from an already parsed
-   refresh token. Takes ownership of the refresh token. */
-grpc_call_credentials *
-grpc_refresh_token_credentials_create_from_auth_refresh_token(
-    grpc_auth_refresh_token token);
-
 /* --- grpc_server_credentials. --- */
 
 typedef struct {
@@ -260,118 +218,19 @@ grpc_server_credentials *grpc_server_credentials_from_arg(const grpc_arg *arg);
 grpc_server_credentials *grpc_find_server_credentials_in_args(
     const grpc_channel_args *args);
 
-/* -- Fake transport security credentials. -- */
-
-/* Creates a fake transport security credentials object for testing. */
-grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
-/* Creates a fake server transport security credentials object for testing. */
-grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
-    void);
-
-/* -- Ssl credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_ssl_config config;
-} grpc_ssl_credentials;
-
-typedef struct {
-  grpc_server_credentials base;
-  grpc_ssl_server_config config;
-} grpc_ssl_server_credentials;
-
-/* -- Channel composite credentials. -- */
-
-typedef struct {
-  grpc_channel_credentials base;
-  grpc_channel_credentials *inner_creds;
-  grpc_call_credentials *call_creds;
-} grpc_composite_channel_credentials;
-
-/* -- Jwt credentials -- */
+/* -- Credentials Metadata Request. -- */
 
 typedef struct {
-  grpc_call_credentials base;
-
-  /* Have a simple cache for now with just 1 entry. We could have a map based on
-     the service_url for a more sophisticated one. */
-  gpr_mu cache_mu;
-  struct {
-    grpc_credentials_md_store *jwt_md;
-    char *service_url;
-    gpr_timespec jwt_expiration;
-  } cached;
-
-  grpc_auth_json_key key;
-  gpr_timespec jwt_lifetime;
-} grpc_service_account_jwt_access_credentials;
-
-/* -- Oauth2TokenFetcher credentials --
-
-   This object is a base for credentials that need to acquire an oauth2 token
-   from an http service. */
-
-typedef struct grpc_credentials_metadata_request
-    grpc_credentials_metadata_request;
+  grpc_call_credentials *creds;
+  grpc_credentials_metadata_cb cb;
+  void *user_data;
+} grpc_credentials_metadata_request;
 
-typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
-                                       grpc_credentials_metadata_request *req,
-                                       grpc_httpcli_context *http_context,
-                                       grpc_pollset *pollset,
-                                       grpc_httpcli_response_cb response_cb,
-                                       gpr_timespec deadline);
+grpc_credentials_metadata_request *grpc_credentials_metadata_request_create(
+    grpc_call_credentials *creds, grpc_credentials_metadata_cb cb,
+    void *user_data);
 
-typedef struct {
-  grpc_call_credentials base;
-  gpr_mu mu;
-  grpc_credentials_md_store *access_token_md;
-  gpr_timespec token_expiration;
-  grpc_httpcli_context httpcli_context;
-  grpc_fetch_oauth2_func fetch_func;
-} grpc_oauth2_token_fetcher_credentials;
-
-/* -- GoogleRefreshToken credentials. -- */
-
-typedef struct {
-  grpc_oauth2_token_fetcher_credentials base;
-  grpc_auth_refresh_token refresh_token;
-} grpc_google_refresh_token_credentials;
-
-/* -- Oauth2 Access Token credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *access_token_md;
-} grpc_access_token_credentials;
-
-/* --  Metadata-only Test credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *md_store;
-  int is_async;
-} grpc_md_only_test_credentials;
-
-/* -- GoogleIAM credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_credentials_md_store *iam_md;
-} grpc_google_iam_credentials;
-
-/* -- Composite credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_call_credentials_array inner;
-} grpc_composite_call_credentials;
-
-/* -- Plugin credentials. -- */
-
-typedef struct {
-  grpc_call_credentials base;
-  grpc_metadata_credentials_plugin plugin;
-  grpc_credentials_md_store *plugin_md;
-} grpc_plugin_credentials;
+void grpc_credentials_metadata_request_destroy(
+    grpc_credentials_metadata_request *r);
 
-#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_metadata.c b/src/core/lib/security/credentials/credentials_metadata.c
similarity index 98%
rename from src/core/lib/security/credentials_metadata.c
rename to src/core/lib/security/credentials/credentials_metadata.c
index bd00194278e84b63b345f49a233e2241e07cc6ba..6a352aab3a6ca2650435973b19f26b497f8936cb 100644
--- a/src/core/lib/security/credentials_metadata.c
+++ b/src/core/lib/security/credentials/credentials_metadata.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <grpc/support/alloc.h>
 
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.c b/src/core/lib/security/credentials/fake/fake_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..54d7cf2581976fbc4b5c95d639dd320fe5b8506f
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.c
@@ -0,0 +1,138 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/iomgr/executor.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+/* -- Fake transport security credentials. -- */
+
+static grpc_security_status fake_transport_security_create_security_connector(
+    grpc_channel_credentials *c, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  *sc = grpc_fake_channel_security_connector_create(call_creds);
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_security_status
+fake_transport_security_server_create_security_connector(
+    grpc_server_credentials *c, grpc_server_security_connector **sc) {
+  *sc = grpc_fake_server_security_connector_create();
+  return GRPC_SECURITY_OK;
+}
+
+static grpc_channel_credentials_vtable
+    fake_transport_security_credentials_vtable = {
+        NULL, fake_transport_security_create_security_connector};
+
+static grpc_server_credentials_vtable
+    fake_transport_security_server_credentials_vtable = {
+        NULL, fake_transport_security_server_create_security_connector};
+
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(
+    void) {
+  grpc_channel_credentials *c = gpr_malloc(sizeof(grpc_channel_credentials));
+  memset(c, 0, sizeof(grpc_channel_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  c->vtable = &fake_transport_security_credentials_vtable;
+  gpr_ref_init(&c->refcount, 1);
+  return c;
+}
+
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void) {
+  grpc_server_credentials *c = gpr_malloc(sizeof(grpc_server_credentials));
+  memset(c, 0, sizeof(grpc_server_credentials));
+  c->type = GRPC_CHANNEL_CREDENTIALS_TYPE_FAKE_TRANSPORT_SECURITY;
+  gpr_ref_init(&c->refcount, 1);
+  c->vtable = &fake_transport_security_server_credentials_vtable;
+  return c;
+}
+
+/* -- Metadata-only test credentials. -- */
+
+static void md_only_test_destruct(grpc_call_credentials *creds) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+  grpc_credentials_md_store_unref(c->md_store);
+}
+
+static void on_simulated_token_fetch_done(grpc_exec_ctx *exec_ctx,
+                                          void *user_data, bool success) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)r->creds;
+  r->cb(exec_ctx, r->user_data, c->md_store->entries, c->md_store->num_entries,
+        GRPC_CREDENTIALS_OK);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void md_only_test_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_md_only_test_credentials *c = (grpc_md_only_test_credentials *)creds;
+
+  if (c->is_async) {
+    grpc_credentials_metadata_request *cb_arg =
+        grpc_credentials_metadata_request_create(creds, cb, user_data);
+    grpc_executor_enqueue(
+        grpc_closure_create(on_simulated_token_fetch_done, cb_arg), true);
+  } else {
+    cb(exec_ctx, user_data, c->md_store->entries, 1, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable md_only_test_vtable = {
+    md_only_test_destruct, md_only_test_get_request_metadata};
+
+grpc_call_credentials *grpc_md_only_test_credentials_create(
+    const char *md_key, const char *md_value, int is_async) {
+  grpc_md_only_test_credentials *c =
+      gpr_malloc(sizeof(grpc_md_only_test_credentials));
+  memset(c, 0, sizeof(grpc_md_only_test_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &md_only_test_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->md_store = grpc_credentials_md_store_create(1);
+  grpc_credentials_md_store_add_cstrings(c->md_store, md_key, md_value);
+  c->is_async = is_async;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/fake/fake_credentials.h b/src/core/lib/security/credentials/fake/fake_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..9cf38084a3df3fdddfa785b1d649d6fef33aee10
--- /dev/null
+++ b/src/core/lib/security/credentials/fake/fake_credentials.h
@@ -0,0 +1,56 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+/* -- Fake transport security credentials. -- */
+
+/* Creates a fake transport security credentials object for testing. */
+grpc_channel_credentials *grpc_fake_transport_security_credentials_create(void);
+
+/* Creates a fake server transport security credentials object for testing. */
+grpc_server_credentials *grpc_fake_transport_security_server_credentials_create(
+    void);
+
+/* --  Metadata-only Test credentials. -- */
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *md_store;
+  int is_async;
+} grpc_md_only_test_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_FAKE_FAKE_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials_posix.c b/src/core/lib/security/credentials/google_default/credentials_posix.c
similarity index 96%
rename from src/core/lib/security/credentials_posix.c
rename to src/core/lib/security/credentials/google_default/credentials_posix.c
index a07de182a0d55fce8e33b6954934e15e44eb9333..42c9d7f997a6237da1b32e9ffc179973e7467f16 100644
--- a/src/core/lib/security/credentials_posix.c
+++ b/src/core/lib/security/credentials/google_default/credentials_posix.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_POSIX_FILE
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/credentials_win32.c b/src/core/lib/security/credentials/google_default/credentials_win32.c
similarity index 96%
rename from src/core/lib/security/credentials_win32.c
rename to src/core/lib/security/credentials/google_default/credentials_win32.c
index d29847af38ad40e1d8051bbe146098c027842d71..cd8b48080a4e6c0b0f26740232a58bb214d03bae 100644
--- a/src/core/lib/security/credentials_win32.c
+++ b/src/core/lib/security/credentials/google_default/credentials_win32.c
@@ -35,7 +35,7 @@
 
 #ifdef GPR_WIN32
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/google_default_credentials.c b/src/core/lib/security/credentials/google_default/google_default_credentials.c
similarity index 97%
rename from src/core/lib/security/google_default_credentials.c
rename to src/core/lib/security/credentials/google_default/google_default_credentials.c
index 236f1d7fa7e8f778d045b858727baad6e3c96fa5..a521d95abcebba408458be3d0e2b925b2ee89122 100644
--- a/src/core/lib/security/google_default_credentials.c
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <string.h>
 
@@ -41,6 +41,8 @@
 
 #include "src/core/lib/http/httpcli.h"
 #include "src/core/lib/http/parser.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/surface/api_trace.h"
diff --git a/src/core/lib/security/credentials/google_default/google_default_credentials.h b/src/core/lib/security/credentials/google_default/google_default_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..fac4377e2c8d1770ad51de3a36a54f299b1948ab
--- /dev/null
+++ b/src/core/lib/security/credentials/google_default/google_default_credentials.h
@@ -0,0 +1,46 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+#define GRPC_GOOGLE_CLOUD_SDK_CONFIG_DIRECTORY "gcloud"
+#define GRPC_GOOGLE_WELL_KNOWN_CREDENTIALS_FILE \
+  "application_default_credentials.json"
+
+void grpc_flush_cached_google_default_credentials(void);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_GOOGLE_DEFAULT_GOOGLE_DEFAULT_CREDENTIALS_H \
+          */
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.c b/src/core/lib/security/credentials/iam/iam_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..89defa7c60bce07bdfbd6fce5b1fbb5fbf04e877
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.c
@@ -0,0 +1,85 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/iam/iam_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void iam_destruct(grpc_call_credentials *creds) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  grpc_credentials_md_store_unref(c->iam_md);
+}
+
+static void iam_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pollset *pollset,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_google_iam_credentials *c = (grpc_google_iam_credentials *)creds;
+  cb(exec_ctx, user_data, c->iam_md->entries, c->iam_md->num_entries,
+     GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable iam_vtable = {iam_destruct,
+                                                  iam_get_request_metadata};
+
+grpc_call_credentials *grpc_google_iam_credentials_create(
+    const char *token, const char *authority_selector, void *reserved) {
+  grpc_google_iam_credentials *c;
+  GRPC_API_TRACE(
+      "grpc_iam_credentials_create(token=%s, authority_selector=%s, "
+      "reserved=%p)",
+      3, (token, authority_selector, reserved));
+  GPR_ASSERT(reserved == NULL);
+  GPR_ASSERT(token != NULL);
+  GPR_ASSERT(authority_selector != NULL);
+  c = gpr_malloc(sizeof(grpc_google_iam_credentials));
+  memset(c, 0, sizeof(grpc_google_iam_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_IAM;
+  c->base.vtable = &iam_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->iam_md = grpc_credentials_md_store_create(2);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORIZATION_TOKEN_METADATA_KEY, token);
+  grpc_credentials_md_store_add_cstrings(
+      c->iam_md, GRPC_IAM_AUTHORITY_SELECTOR_METADATA_KEY, authority_selector);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/iam/iam_credentials.h b/src/core/lib/security/credentials/iam/iam_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..af54faa5868e6120239ef8ddefc67bf1a2bcdb6a
--- /dev/null
+++ b/src/core/lib/security/credentials/iam/iam_credentials.h
@@ -0,0 +1,44 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *iam_md;
+} grpc_google_iam_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_IAM_IAM_CREDENTIALS_H */
diff --git a/src/core/lib/security/json_token.c b/src/core/lib/security/credentials/jwt/json_token.c
similarity index 74%
rename from src/core/lib/security/json_token.c
rename to src/core/lib/security/credentials/jwt/json_token.c
index d5bc2c8d6081d3b0c8567c3e1c89496922ff447e..354c13133ef8bffcb91a49f05357a61b72cd5e83 100644
--- a/src/core/lib/security/json_token.c
+++ b/src/core/lib/security/credentials/jwt/json_token.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
 #include <string.h>
 
@@ -39,7 +39,8 @@
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
+#include "src/core/lib/security/util/json_util.h"
 #include "src/core/lib/support/string.h"
 
 #include <openssl/bio.h>
@@ -66,28 +67,6 @@ static grpc_jwt_encode_and_sign_override g_jwt_encode_and_sign_override = NULL;
 
 /* --- grpc_auth_json_key. --- */
 
-static const char *json_get_string_property(const grpc_json *json,
-                                            const char *prop_name) {
-  grpc_json *child;
-  for (child = json->child; child != NULL; child = child->next) {
-    if (strcmp(child->key, prop_name) == 0) break;
-  }
-  if (child == NULL || child->type != GRPC_JSON_STRING) {
-    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
-    return NULL;
-  }
-  return child->value;
-}
-
-static int set_json_key_string_property(const grpc_json *json,
-                                        const char *prop_name,
-                                        char **json_key_field) {
-  const char *prop_value = json_get_string_property(json, prop_name);
-  if (prop_value == NULL) return 0;
-  *json_key_field = gpr_strdup(prop_value);
-  return 1;
-}
-
 int grpc_auth_json_key_is_valid(const grpc_auth_json_key *json_key) {
   return (json_key != NULL) &&
          strcmp(json_key->type, GRPC_AUTH_JSON_TYPE_INVALID);
@@ -106,22 +85,22 @@ grpc_auth_json_key grpc_auth_json_key_create_from_json(const grpc_json *json) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "type");
+  prop_value = grpc_json_get_string_property(json, "type");
   if (prop_value == NULL ||
       strcmp(prop_value, GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT)) {
     goto end;
   }
   result.type = GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT;
 
-  if (!set_json_key_string_property(json, "private_key_id",
-                                    &result.private_key_id) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "client_email",
-                                    &result.client_email)) {
+  if (!grpc_copy_json_string_property(json, "private_key_id",
+                                      &result.private_key_id) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "client_email",
+                                      &result.client_email)) {
     goto end;
   }
 
-  prop_value = json_get_string_property(json, "private_key");
+  prop_value = grpc_json_get_string_property(json, "private_key");
   if (prop_value == NULL) {
     goto end;
   }
@@ -339,73 +318,3 @@ void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func) {
   g_jwt_encode_and_sign_override = func;
 }
-
-/* --- grpc_auth_refresh_token --- */
-
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token) {
-  return (refresh_token != NULL) &&
-         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json) {
-  grpc_auth_refresh_token result;
-  const char *prop_value;
-  int success = 0;
-
-  memset(&result, 0, sizeof(grpc_auth_refresh_token));
-  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (json == NULL) {
-    gpr_log(GPR_ERROR, "Invalid json.");
-    goto end;
-  }
-
-  prop_value = json_get_string_property(json, "type");
-  if (prop_value == NULL ||
-      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
-    goto end;
-  }
-  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
-
-  if (!set_json_key_string_property(json, "client_secret",
-                                    &result.client_secret) ||
-      !set_json_key_string_property(json, "client_id", &result.client_id) ||
-      !set_json_key_string_property(json, "refresh_token",
-                                    &result.refresh_token)) {
-    goto end;
-  }
-  success = 1;
-
-end:
-  if (!success) grpc_auth_refresh_token_destruct(&result);
-  return result;
-}
-
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string) {
-  char *scratchpad = gpr_strdup(json_string);
-  grpc_json *json = grpc_json_parse_string(scratchpad);
-  grpc_auth_refresh_token result =
-      grpc_auth_refresh_token_create_from_json(json);
-  if (json != NULL) grpc_json_destroy(json);
-  gpr_free(scratchpad);
-  return result;
-}
-
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
-  if (refresh_token == NULL) return;
-  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
-  if (refresh_token->client_id != NULL) {
-    gpr_free(refresh_token->client_id);
-    refresh_token->client_id = NULL;
-  }
-  if (refresh_token->client_secret != NULL) {
-    gpr_free(refresh_token->client_secret);
-    refresh_token->client_secret = NULL;
-  }
-  if (refresh_token->refresh_token != NULL) {
-    gpr_free(refresh_token->refresh_token);
-    refresh_token->refresh_token = NULL;
-  }
-}
diff --git a/src/core/lib/security/json_token.h b/src/core/lib/security/credentials/jwt/json_token.h
similarity index 74%
rename from src/core/lib/security/json_token.h
rename to src/core/lib/security/credentials/jwt/json_token.h
index 123fa652fd808079961589fa2887f2e7bf940580..07fc5bf0e012b7a07e42309f38c18221a8aa2758 100644
--- a/src/core/lib/security/json_token.h
+++ b/src/core/lib/security/credentials/jwt/json_token.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
-#define GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H
 
 #include <grpc/support/slice.h>
 #include <openssl/rsa.h>
@@ -43,10 +43,6 @@
 
 #define GRPC_JWT_OAUTH2_AUDIENCE "https://www.googleapis.com/oauth2/v3/token"
 
-#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
-#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
-#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
-
 /* --- auth_json_key parsing. --- */
 
 typedef struct {
@@ -89,30 +85,4 @@ typedef char *(*grpc_jwt_encode_and_sign_override)(
 void grpc_jwt_encode_and_sign_set_override(
     grpc_jwt_encode_and_sign_override func);
 
-/* --- auth_refresh_token parsing. --- */
-
-typedef struct {
-  const char *type;
-  char *client_id;
-  char *client_secret;
-  char *refresh_token;
-} grpc_auth_refresh_token;
-
-/* Returns 1 if the object is valid, 0 otherwise. */
-int grpc_auth_refresh_token_is_valid(
-    const grpc_auth_refresh_token *refresh_token);
-
-/* Creates a refresh token object from string. Returns an invalid object if a
-   parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
-    const char *json_string);
-
-/* Creates a refresh token object from parsed json. Returns an invalid object if
-   a parsing error has been encountered. */
-grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
-    const grpc_json *json);
-
-/* Destructs the object. */
-void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
-
-#endif /* GRPC_CORE_LIB_SECURITY_JSON_TOKEN_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JSON_TOKEN_H */
diff --git a/src/core/lib/security/credentials/jwt/jwt_credentials.c b/src/core/lib/security/credentials/jwt/jwt_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..8755a96af4fbaf83c5da22d546efb2ca7a0057a7
--- /dev/null
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.c
@@ -0,0 +1,160 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+static void jwt_reset_cache(grpc_service_account_jwt_access_credentials *c) {
+  if (c->cached.jwt_md != NULL) {
+    grpc_credentials_md_store_unref(c->cached.jwt_md);
+    c->cached.jwt_md = NULL;
+  }
+  if (c->cached.service_url != NULL) {
+    gpr_free(c->cached.service_url);
+    c->cached.service_url = NULL;
+  }
+  c->cached.jwt_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+}
+
+static void jwt_destruct(grpc_call_credentials *creds) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  grpc_auth_json_key_destruct(&c->key);
+  jwt_reset_cache(c);
+  gpr_mu_destroy(&c->cache_mu);
+}
+
+static void jwt_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                     grpc_call_credentials *creds,
+                                     grpc_pollset *pollset,
+                                     grpc_auth_metadata_context context,
+                                     grpc_credentials_metadata_cb cb,
+                                     void *user_data) {
+  grpc_service_account_jwt_access_credentials *c =
+      (grpc_service_account_jwt_access_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+
+  /* See if we can return a cached jwt. */
+  grpc_credentials_md_store *jwt_md = NULL;
+  {
+    gpr_mu_lock(&c->cache_mu);
+    if (c->cached.service_url != NULL &&
+        strcmp(c->cached.service_url, context.service_url) == 0 &&
+        c->cached.jwt_md != NULL &&
+        (gpr_time_cmp(gpr_time_sub(c->cached.jwt_expiration,
+                                   gpr_now(GPR_CLOCK_REALTIME)),
+                      refresh_threshold) > 0)) {
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md == NULL) {
+    char *jwt = NULL;
+    /* Generate a new jwt. */
+    gpr_mu_lock(&c->cache_mu);
+    jwt_reset_cache(c);
+    jwt = grpc_jwt_encode_and_sign(&c->key, context.service_url,
+                                   c->jwt_lifetime, NULL);
+    if (jwt != NULL) {
+      char *md_value;
+      gpr_asprintf(&md_value, "Bearer %s", jwt);
+      gpr_free(jwt);
+      c->cached.jwt_expiration =
+          gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), c->jwt_lifetime);
+      c->cached.service_url = gpr_strdup(context.service_url);
+      c->cached.jwt_md = grpc_credentials_md_store_create(1);
+      grpc_credentials_md_store_add_cstrings(
+          c->cached.jwt_md, GRPC_AUTHORIZATION_METADATA_KEY, md_value);
+      gpr_free(md_value);
+      jwt_md = grpc_credentials_md_store_ref(c->cached.jwt_md);
+    }
+    gpr_mu_unlock(&c->cache_mu);
+  }
+
+  if (jwt_md != NULL) {
+    cb(exec_ctx, user_data, jwt_md->entries, jwt_md->num_entries,
+       GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(jwt_md);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  }
+}
+
+static grpc_call_credentials_vtable jwt_vtable = {jwt_destruct,
+                                                  jwt_get_request_metadata};
+
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime) {
+  grpc_service_account_jwt_access_credentials *c;
+  if (!grpc_auth_json_key_is_valid(&key)) {
+    gpr_log(GPR_ERROR, "Invalid input for jwt credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_service_account_jwt_access_credentials));
+  memset(c, 0, sizeof(grpc_service_account_jwt_access_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_JWT;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &jwt_vtable;
+  c->key = key;
+  c->jwt_lifetime = token_lifetime;
+  gpr_mu_init(&c->cache_mu);
+  jwt_reset_cache(c);
+  return &c->base;
+}
+
+grpc_call_credentials *grpc_service_account_jwt_access_credentials_create(
+    const char *json_key, gpr_timespec token_lifetime, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_service_account_jwt_access_credentials_create("
+      "json_key=%s, "
+      "token_lifetime="
+      "gpr_timespec { tv_sec: %lld, tv_nsec: %d, clock_type: %d }, "
+      "reserved=%p)",
+      5,
+      (json_key, (long long)token_lifetime.tv_sec, (int)token_lifetime.tv_nsec,
+       (int)token_lifetime.clock_type, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+      grpc_auth_json_key_create_from_string(json_key), token_lifetime);
+}
diff --git a/test/cpp/qps/async_streaming_ping_pong_test.cc b/src/core/lib/security/credentials/jwt/jwt_credentials.h
similarity index 56%
rename from test/cpp/qps/async_streaming_ping_pong_test.cc
rename to src/core/lib/security/credentials/jwt/jwt_credentials.h
index 4b6bae0d5cff0ed494f6f305ed06f774c903f882..d5726061794dbf54bc8e31c4f306394cbc70dc12 100644
--- a/test/cpp/qps/async_streaming_ping_pong_test.cc
+++ b/src/core/lib/security/credentials/jwt/jwt_credentials.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,48 +31,32 @@
  *
  */
 
-#include <set>
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H
 
-#include <grpc/support/log.h>
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
+typedef struct {
+  grpc_call_credentials base;
 
-namespace grpc {
-namespace testing {
+  // Have a simple cache for now with just 1 entry. We could have a map based on
+  // the service_url for a more sophisticated one.
+  gpr_mu cache_mu;
+  struct {
+    grpc_credentials_md_store *jwt_md;
+    char *service_url;
+    gpr_timespec jwt_expiration;
+  } cached;
 
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
+  grpc_auth_json_key key;
+  gpr_timespec jwt_lifetime;
+} grpc_service_account_jwt_access_credentials;
 
-static void RunAsyncStreamingPingPong() {
-  gpr_log(GPR_INFO, "Running Async Streaming Ping Pong");
+// Private constructor for jwt credentials from an already parsed json key.
+// Takes ownership of the key.
+grpc_call_credentials *
+grpc_service_account_jwt_access_credentials_create_from_auth_json_key(
+    grpc_auth_json_key key, gpr_timespec token_lifetime);
 
-  ClientConfig client_config;
-  client_config.set_client_type(ASYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1);
-  client_config.set_client_channels(1);
-  client_config.set_async_client_threads(1);
-  client_config.set_rpc_type(STREAMING);
-  client_config.mutable_load_params()->mutable_closed_loop();
-
-  ServerConfig server_config;
-  server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_async_server_threads(1);
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPS(*result);
-  GetReporter()->ReportLatency(*result);
-}
-
-}  // namespace testing
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
-
-  grpc::testing::RunAsyncStreamingPingPong();
-  return 0;
-}
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_CREDENTIALS_H */
diff --git a/src/core/lib/security/jwt_verifier.c b/src/core/lib/security/credentials/jwt/jwt_verifier.c
similarity index 99%
rename from src/core/lib/security/jwt_verifier.c
rename to src/core/lib/security/credentials/jwt/jwt_verifier.c
index 0e012294de76cd15f5dbafed357ca23e3c324066..cd6c7ce39210e381e7309942f5cc2cde82cd1c3e 100644
--- a/src/core/lib/security/jwt_verifier.c
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.c
@@ -31,13 +31,13 @@
  *
  */
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 #include <limits.h>
 #include <string.h>
 
 #include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 #include "src/core/lib/tsi/ssl_types.h"
 
 #include <grpc/support/alloc.h>
diff --git a/src/core/lib/security/jwt_verifier.h b/src/core/lib/security/credentials/jwt/jwt_verifier.h
similarity index 96%
rename from src/core/lib/security/jwt_verifier.h
rename to src/core/lib/security/credentials/jwt/jwt_verifier.h
index 98a4f6b1162880f19ffb02fa06d2589821adeb03..b0f6d1c240da6f36f88cfe9300baad08cee1cebb 100644
--- a/src/core/lib/security/jwt_verifier.h
+++ b/src/core/lib/security/credentials/jwt/jwt_verifier.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
-#define GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H
 
 #include "src/core/lib/iomgr/pollset.h"
 #include "src/core/lib/json/json.h"
@@ -133,4 +133,4 @@ grpc_jwt_claims *grpc_jwt_claims_from_json(grpc_json *json, gpr_slice buffer);
 grpc_jwt_verifier_status grpc_jwt_claims_check(const grpc_jwt_claims *claims,
                                                const char *audience);
 
-#endif /* GRPC_CORE_LIB_SECURITY_JWT_VERIFIER_H */
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_JWT_JWT_VERIFIER_H */
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.c b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..973c6e1d17f2c7c70be643b46cc9d44c1fc1c652
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.c
@@ -0,0 +1,428 @@
+/*
+ *
+ * Copyright 2015, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/security/util/json_util.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+//
+// Auth Refresh Token.
+//
+
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token) {
+  return (refresh_token != NULL) &&
+         strcmp(refresh_token->type, GRPC_AUTH_JSON_TYPE_INVALID);
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json) {
+  grpc_auth_refresh_token result;
+  const char *prop_value;
+  int success = 0;
+
+  memset(&result, 0, sizeof(grpc_auth_refresh_token));
+  result.type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (json == NULL) {
+    gpr_log(GPR_ERROR, "Invalid json.");
+    goto end;
+  }
+
+  prop_value = grpc_json_get_string_property(json, "type");
+  if (prop_value == NULL ||
+      strcmp(prop_value, GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER)) {
+    goto end;
+  }
+  result.type = GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER;
+
+  if (!grpc_copy_json_string_property(json, "client_secret",
+                                      &result.client_secret) ||
+      !grpc_copy_json_string_property(json, "client_id", &result.client_id) ||
+      !grpc_copy_json_string_property(json, "refresh_token",
+                                      &result.refresh_token)) {
+    goto end;
+  }
+  success = 1;
+
+end:
+  if (!success) grpc_auth_refresh_token_destruct(&result);
+  return result;
+}
+
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string) {
+  char *scratchpad = gpr_strdup(json_string);
+  grpc_json *json = grpc_json_parse_string(scratchpad);
+  grpc_auth_refresh_token result =
+      grpc_auth_refresh_token_create_from_json(json);
+  if (json != NULL) grpc_json_destroy(json);
+  gpr_free(scratchpad);
+  return result;
+}
+
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token) {
+  if (refresh_token == NULL) return;
+  refresh_token->type = GRPC_AUTH_JSON_TYPE_INVALID;
+  if (refresh_token->client_id != NULL) {
+    gpr_free(refresh_token->client_id);
+    refresh_token->client_id = NULL;
+  }
+  if (refresh_token->client_secret != NULL) {
+    gpr_free(refresh_token->client_secret);
+    refresh_token->client_secret = NULL;
+  }
+  if (refresh_token->refresh_token != NULL) {
+    gpr_free(refresh_token->refresh_token);
+    refresh_token->refresh_token = NULL;
+  }
+}
+
+//
+// Oauth2 Token Fetcher credentials.
+//
+
+static void oauth2_token_fetcher_destruct(grpc_call_credentials *creds) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+  gpr_mu_destroy(&c->mu);
+  grpc_httpcli_context_destroy(&c->httpcli_context);
+}
+
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const grpc_http_response *response, grpc_credentials_md_store **token_md,
+    gpr_timespec *token_lifetime) {
+  char *null_terminated_body = NULL;
+  char *new_access_token = NULL;
+  grpc_credentials_status status = GRPC_CREDENTIALS_OK;
+  grpc_json *json = NULL;
+
+  if (response == NULL) {
+    gpr_log(GPR_ERROR, "Received NULL response.");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  }
+
+  if (response->body_length > 0) {
+    null_terminated_body = gpr_malloc(response->body_length + 1);
+    null_terminated_body[response->body_length] = '\0';
+    memcpy(null_terminated_body, response->body, response->body_length);
+  }
+
+  if (response->status != 200) {
+    gpr_log(GPR_ERROR, "Call to http server ended with error %d [%s].",
+            response->status,
+            null_terminated_body != NULL ? null_terminated_body : "");
+    status = GRPC_CREDENTIALS_ERROR;
+    goto end;
+  } else {
+    grpc_json *access_token = NULL;
+    grpc_json *token_type = NULL;
+    grpc_json *expires_in = NULL;
+    grpc_json *ptr;
+    json = grpc_json_parse_string(null_terminated_body);
+    if (json == NULL) {
+      gpr_log(GPR_ERROR, "Could not parse JSON from %s", null_terminated_body);
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (json->type != GRPC_JSON_OBJECT) {
+      gpr_log(GPR_ERROR, "Response should be a JSON object");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    for (ptr = json->child; ptr; ptr = ptr->next) {
+      if (strcmp(ptr->key, "access_token") == 0) {
+        access_token = ptr;
+      } else if (strcmp(ptr->key, "token_type") == 0) {
+        token_type = ptr;
+      } else if (strcmp(ptr->key, "expires_in") == 0) {
+        expires_in = ptr;
+      }
+    }
+    if (access_token == NULL || access_token->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid access_token in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (token_type == NULL || token_type->type != GRPC_JSON_STRING) {
+      gpr_log(GPR_ERROR, "Missing or invalid token_type in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    if (expires_in == NULL || expires_in->type != GRPC_JSON_NUMBER) {
+      gpr_log(GPR_ERROR, "Missing or invalid expires_in in JSON.");
+      status = GRPC_CREDENTIALS_ERROR;
+      goto end;
+    }
+    gpr_asprintf(&new_access_token, "%s %s", token_type->value,
+                 access_token->value);
+    token_lifetime->tv_sec = strtol(expires_in->value, NULL, 10);
+    token_lifetime->tv_nsec = 0;
+    token_lifetime->clock_type = GPR_TIMESPAN;
+    if (*token_md != NULL) grpc_credentials_md_store_unref(*token_md);
+    *token_md = grpc_credentials_md_store_create(1);
+    grpc_credentials_md_store_add_cstrings(
+        *token_md, GRPC_AUTHORIZATION_METADATA_KEY, new_access_token);
+    status = GRPC_CREDENTIALS_OK;
+  }
+
+end:
+  if (status != GRPC_CREDENTIALS_OK && (*token_md != NULL)) {
+    grpc_credentials_md_store_unref(*token_md);
+    *token_md = NULL;
+  }
+  if (null_terminated_body != NULL) gpr_free(null_terminated_body);
+  if (new_access_token != NULL) gpr_free(new_access_token);
+  if (json != NULL) grpc_json_destroy(json);
+  return status;
+}
+
+static void on_oauth2_token_fetcher_http_response(
+    grpc_exec_ctx *exec_ctx, void *user_data,
+    const grpc_http_response *response) {
+  grpc_credentials_metadata_request *r =
+      (grpc_credentials_metadata_request *)user_data;
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)r->creds;
+  gpr_timespec token_lifetime;
+  grpc_credentials_status status;
+
+  gpr_mu_lock(&c->mu);
+  status = grpc_oauth2_token_fetcher_credentials_parse_server_response(
+      response, &c->access_token_md, &token_lifetime);
+  if (status == GRPC_CREDENTIALS_OK) {
+    c->token_expiration =
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), token_lifetime);
+    r->cb(exec_ctx, r->user_data, c->access_token_md->entries,
+          c->access_token_md->num_entries, status);
+  } else {
+    c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+    r->cb(exec_ctx, r->user_data, NULL, 0, status);
+  }
+  gpr_mu_unlock(&c->mu);
+  grpc_credentials_metadata_request_destroy(r);
+}
+
+static void oauth2_token_fetcher_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      (grpc_oauth2_token_fetcher_credentials *)creds;
+  gpr_timespec refresh_threshold = gpr_time_from_seconds(
+      GRPC_SECURE_TOKEN_REFRESH_THRESHOLD_SECS, GPR_TIMESPAN);
+  grpc_credentials_md_store *cached_access_token_md = NULL;
+  {
+    gpr_mu_lock(&c->mu);
+    if (c->access_token_md != NULL &&
+        (gpr_time_cmp(
+             gpr_time_sub(c->token_expiration, gpr_now(GPR_CLOCK_REALTIME)),
+             refresh_threshold) > 0)) {
+      cached_access_token_md =
+          grpc_credentials_md_store_ref(c->access_token_md);
+    }
+    gpr_mu_unlock(&c->mu);
+  }
+  if (cached_access_token_md != NULL) {
+    cb(exec_ctx, user_data, cached_access_token_md->entries,
+       cached_access_token_md->num_entries, GRPC_CREDENTIALS_OK);
+    grpc_credentials_md_store_unref(cached_access_token_md);
+  } else {
+    c->fetch_func(
+        exec_ctx,
+        grpc_credentials_metadata_request_create(creds, cb, user_data),
+        &c->httpcli_context, pollset, on_oauth2_token_fetcher_http_response,
+        gpr_time_add(gpr_now(GPR_CLOCK_REALTIME), refresh_threshold));
+  }
+}
+
+static void init_oauth2_token_fetcher(grpc_oauth2_token_fetcher_credentials *c,
+                                      grpc_fetch_oauth2_func fetch_func) {
+  memset(c, 0, sizeof(grpc_oauth2_token_fetcher_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  gpr_ref_init(&c->base.refcount, 1);
+  gpr_mu_init(&c->mu);
+  c->token_expiration = gpr_inf_past(GPR_CLOCK_REALTIME);
+  c->fetch_func = fetch_func;
+  grpc_httpcli_context_init(&c->httpcli_context);
+}
+
+//
+//  Google Compute Engine credentials.
+//
+
+static grpc_call_credentials_vtable compute_engine_vtable = {
+    oauth2_token_fetcher_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void compute_engine_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_http_header header = {"Metadata-Flavor", "Google"};
+  grpc_httpcli_request request;
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_COMPUTE_ENGINE_METADATA_HOST;
+  request.http.path = GRPC_COMPUTE_ENGINE_METADATA_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  grpc_httpcli_get(exec_ctx, httpcli_context, pollset, &request, deadline,
+                   response_cb, metadata_req);
+}
+
+grpc_call_credentials *grpc_google_compute_engine_credentials_create(
+    void *reserved) {
+  grpc_oauth2_token_fetcher_credentials *c =
+      gpr_malloc(sizeof(grpc_oauth2_token_fetcher_credentials));
+  GRPC_API_TRACE("grpc_compute_engine_credentials_create(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  init_oauth2_token_fetcher(c, compute_engine_fetch_oauth2);
+  c->base.vtable = &compute_engine_vtable;
+  return &c->base;
+}
+
+//
+// Google Refresh Token credentials.
+//
+
+static void refresh_token_destruct(grpc_call_credentials *creds) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)creds;
+  grpc_auth_refresh_token_destruct(&c->refresh_token);
+  oauth2_token_fetcher_destruct(&c->base.base);
+}
+
+static grpc_call_credentials_vtable refresh_token_vtable = {
+    refresh_token_destruct, oauth2_token_fetcher_get_request_metadata};
+
+static void refresh_token_fetch_oauth2(
+    grpc_exec_ctx *exec_ctx, grpc_credentials_metadata_request *metadata_req,
+    grpc_httpcli_context *httpcli_context, grpc_pollset *pollset,
+    grpc_httpcli_response_cb response_cb, gpr_timespec deadline) {
+  grpc_google_refresh_token_credentials *c =
+      (grpc_google_refresh_token_credentials *)metadata_req->creds;
+  grpc_http_header header = {"Content-Type",
+                             "application/x-www-form-urlencoded"};
+  grpc_httpcli_request request;
+  char *body = NULL;
+  gpr_asprintf(&body, GRPC_REFRESH_TOKEN_POST_BODY_FORMAT_STRING,
+               c->refresh_token.client_id, c->refresh_token.client_secret,
+               c->refresh_token.refresh_token);
+  memset(&request, 0, sizeof(grpc_httpcli_request));
+  request.host = GRPC_GOOGLE_OAUTH2_SERVICE_HOST;
+  request.http.path = GRPC_GOOGLE_OAUTH2_SERVICE_TOKEN_PATH;
+  request.http.hdr_count = 1;
+  request.http.hdrs = &header;
+  request.handshaker = &grpc_httpcli_ssl;
+  grpc_httpcli_post(exec_ctx, httpcli_context, pollset, &request, body,
+                    strlen(body), deadline, response_cb, metadata_req);
+  gpr_free(body);
+}
+
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token refresh_token) {
+  grpc_google_refresh_token_credentials *c;
+  if (!grpc_auth_refresh_token_is_valid(&refresh_token)) {
+    gpr_log(GPR_ERROR, "Invalid input for refresh token credentials creation");
+    return NULL;
+  }
+  c = gpr_malloc(sizeof(grpc_google_refresh_token_credentials));
+  memset(c, 0, sizeof(grpc_google_refresh_token_credentials));
+  init_oauth2_token_fetcher(&c->base, refresh_token_fetch_oauth2);
+  c->base.base.vtable = &refresh_token_vtable;
+  c->refresh_token = refresh_token;
+  return &c->base.base;
+}
+
+grpc_call_credentials *grpc_google_refresh_token_credentials_create(
+    const char *json_refresh_token, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_refresh_token_credentials_create(json_refresh_token=%s, "
+      "reserved=%p)",
+      2, (json_refresh_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  return grpc_refresh_token_credentials_create_from_auth_refresh_token(
+      grpc_auth_refresh_token_create_from_string(json_refresh_token));
+}
+
+//
+// Oauth2 Access Token credentials.
+//
+
+static void access_token_destruct(grpc_call_credentials *creds) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  grpc_credentials_md_store_unref(c->access_token_md);
+}
+
+static void access_token_get_request_metadata(
+    grpc_exec_ctx *exec_ctx, grpc_call_credentials *creds,
+    grpc_pollset *pollset, grpc_auth_metadata_context context,
+    grpc_credentials_metadata_cb cb, void *user_data) {
+  grpc_access_token_credentials *c = (grpc_access_token_credentials *)creds;
+  cb(exec_ctx, user_data, c->access_token_md->entries, 1, GRPC_CREDENTIALS_OK);
+}
+
+static grpc_call_credentials_vtable access_token_vtable = {
+    access_token_destruct, access_token_get_request_metadata};
+
+grpc_call_credentials *grpc_access_token_credentials_create(
+    const char *access_token, void *reserved) {
+  grpc_access_token_credentials *c =
+      gpr_malloc(sizeof(grpc_access_token_credentials));
+  char *token_md_value;
+  GRPC_API_TRACE(
+      "grpc_access_token_credentials_create(access_token=%s, "
+      "reserved=%p)",
+      2, (access_token, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_access_token_credentials));
+  c->base.type = GRPC_CALL_CREDENTIALS_TYPE_OAUTH2;
+  c->base.vtable = &access_token_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->access_token_md = grpc_credentials_md_store_create(1);
+  gpr_asprintf(&token_md_value, "Bearer %s", access_token);
+  grpc_credentials_md_store_add_cstrings(
+      c->access_token_md, GRPC_AUTHORIZATION_METADATA_KEY, token_md_value);
+  gpr_free(token_md_value);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/oauth2/oauth2_credentials.h b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..04915b333f457efd0a525c7a9ab45fdf1466528e
--- /dev/null
+++ b/src/core/lib/security/credentials/oauth2/oauth2_credentials.h
@@ -0,0 +1,109 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H
+
+#include "src/core/lib/json/json.h"
+#include "src/core/lib/security/credentials/credentials.h"
+
+// auth_refresh_token parsing.
+typedef struct {
+  const char *type;
+  char *client_id;
+  char *client_secret;
+  char *refresh_token;
+} grpc_auth_refresh_token;
+
+/// Returns 1 if the object is valid, 0 otherwise.
+int grpc_auth_refresh_token_is_valid(
+    const grpc_auth_refresh_token *refresh_token);
+
+/// Creates a refresh token object from string. Returns an invalid object if a
+/// parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_string(
+    const char *json_string);
+
+/// Creates a refresh token object from parsed json. Returns an invalid object
+/// if a parsing error has been encountered.
+grpc_auth_refresh_token grpc_auth_refresh_token_create_from_json(
+    const grpc_json *json);
+
+/// Destructs the object.
+void grpc_auth_refresh_token_destruct(grpc_auth_refresh_token *refresh_token);
+
+// -- Oauth2 Token Fetcher credentials --
+//
+//  This object is a base for credentials that need to acquire an oauth2 token
+//  from an http service.
+
+typedef void (*grpc_fetch_oauth2_func)(grpc_exec_ctx *exec_ctx,
+                                       grpc_credentials_metadata_request *req,
+                                       grpc_httpcli_context *http_context,
+                                       grpc_pollset *pollset,
+                                       grpc_httpcli_response_cb response_cb,
+                                       gpr_timespec deadline);
+typedef struct {
+  grpc_call_credentials base;
+  gpr_mu mu;
+  grpc_credentials_md_store *access_token_md;
+  gpr_timespec token_expiration;
+  grpc_httpcli_context httpcli_context;
+  grpc_fetch_oauth2_func fetch_func;
+} grpc_oauth2_token_fetcher_credentials;
+
+// Google refresh token credentials.
+typedef struct {
+  grpc_oauth2_token_fetcher_credentials base;
+  grpc_auth_refresh_token refresh_token;
+} grpc_google_refresh_token_credentials;
+
+// Access token credentials.
+typedef struct {
+  grpc_call_credentials base;
+  grpc_credentials_md_store *access_token_md;
+} grpc_access_token_credentials;
+
+// Private constructor for refresh token credentials from an already parsed
+// refresh token. Takes ownership of the refresh token.
+grpc_call_credentials *
+grpc_refresh_token_credentials_create_from_auth_refresh_token(
+    grpc_auth_refresh_token token);
+
+// Exposed for testing only.
+grpc_credentials_status
+grpc_oauth2_token_fetcher_credentials_parse_server_response(
+    const struct grpc_http_response *response,
+    grpc_credentials_md_store **token_md, gpr_timespec *token_lifetime);
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_OAUTH2_OAUTH2_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.c b/src/core/lib/security/credentials/plugin/plugin_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..bae357321eaa7bf8daabddf04bf8875c2da5c74e
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.c
@@ -0,0 +1,129 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/plugin/plugin_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <grpc/support/sync.h>
+
+typedef struct {
+  void *user_data;
+  grpc_credentials_metadata_cb cb;
+} grpc_metadata_plugin_request;
+
+static void plugin_destruct(grpc_call_credentials *creds) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.state != NULL && c->plugin.destroy != NULL) {
+    c->plugin.destroy(c->plugin.state);
+  }
+}
+
+static void plugin_md_request_metadata_ready(void *request,
+                                             const grpc_metadata *md,
+                                             size_t num_md,
+                                             grpc_status_code status,
+                                             const char *error_details) {
+  /* called from application code */
+  grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
+  grpc_metadata_plugin_request *r = (grpc_metadata_plugin_request *)request;
+  if (status != GRPC_STATUS_OK) {
+    if (error_details != NULL) {
+      gpr_log(GPR_ERROR, "Getting metadata from plugin failed with error: %s",
+              error_details);
+    }
+    r->cb(&exec_ctx, r->user_data, NULL, 0, GRPC_CREDENTIALS_ERROR);
+  } else {
+    size_t i;
+    grpc_credentials_md *md_array = NULL;
+    if (num_md > 0) {
+      md_array = gpr_malloc(num_md * sizeof(grpc_credentials_md));
+      for (i = 0; i < num_md; i++) {
+        md_array[i].key = gpr_slice_from_copied_string(md[i].key);
+        md_array[i].value =
+            gpr_slice_from_copied_buffer(md[i].value, md[i].value_length);
+      }
+    }
+    r->cb(&exec_ctx, r->user_data, md_array, num_md, GRPC_CREDENTIALS_OK);
+    if (md_array != NULL) {
+      for (i = 0; i < num_md; i++) {
+        gpr_slice_unref(md_array[i].key);
+        gpr_slice_unref(md_array[i].value);
+      }
+      gpr_free(md_array);
+    }
+  }
+  gpr_free(r);
+  grpc_exec_ctx_finish(&exec_ctx);
+}
+
+static void plugin_get_request_metadata(grpc_exec_ctx *exec_ctx,
+                                        grpc_call_credentials *creds,
+                                        grpc_pollset *pollset,
+                                        grpc_auth_metadata_context context,
+                                        grpc_credentials_metadata_cb cb,
+                                        void *user_data) {
+  grpc_plugin_credentials *c = (grpc_plugin_credentials *)creds;
+  if (c->plugin.get_metadata != NULL) {
+    grpc_metadata_plugin_request *request = gpr_malloc(sizeof(*request));
+    memset(request, 0, sizeof(*request));
+    request->user_data = user_data;
+    request->cb = cb;
+    c->plugin.get_metadata(c->plugin.state, context,
+                           plugin_md_request_metadata_ready, request);
+  } else {
+    cb(exec_ctx, user_data, NULL, 0, GRPC_CREDENTIALS_OK);
+  }
+}
+
+static grpc_call_credentials_vtable plugin_vtable = {
+    plugin_destruct, plugin_get_request_metadata};
+
+grpc_call_credentials *grpc_metadata_credentials_create_from_plugin(
+    grpc_metadata_credentials_plugin plugin, void *reserved) {
+  grpc_plugin_credentials *c = gpr_malloc(sizeof(*c));
+  GRPC_API_TRACE("grpc_metadata_credentials_create_from_plugin(reserved=%p)", 1,
+                 (reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(*c));
+  c->base.type = plugin.type;
+  c->base.vtable = &plugin_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->plugin = plugin;
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/plugin/plugin_credentials.h b/src/core/lib/security/credentials/plugin/plugin_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..89073cb3d1b75ab0cb2fc41c60ab3266f660f07a
--- /dev/null
+++ b/src/core/lib/security/credentials/plugin/plugin_credentials.h
@@ -0,0 +1,45 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_call_credentials base;
+  grpc_metadata_credentials_plugin plugin;
+  grpc_credentials_md_store *plugin_md;
+} grpc_plugin_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_PLUGIN_PLUGIN_CREDENTIALS_H */
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.c b/src/core/lib/security/credentials/ssl/ssl_credentials.c
new file mode 100644
index 0000000000000000000000000000000000000000..545bca9d98b6886bfc5f506bb27eba91e8622183
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.c
@@ -0,0 +1,240 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+#include "src/core/lib/security/credentials/ssl/ssl_credentials.h"
+
+#include <string.h>
+
+#include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/http_client_filter.h"
+#include "src/core/lib/surface/api_trace.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+//
+// Utils
+//
+
+static void ssl_copy_key_material(const char *input, unsigned char **output,
+                                  size_t *output_size) {
+  *output_size = strlen(input);
+  *output = gpr_malloc(*output_size);
+  memcpy(*output, input, *output_size);
+}
+
+//
+// SSL Channel Credentials.
+//
+
+static void ssl_destruct(grpc_channel_credentials *creds) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+  if (c->config.pem_private_key != NULL) gpr_free(c->config.pem_private_key);
+  if (c->config.pem_cert_chain != NULL) gpr_free(c->config.pem_cert_chain);
+}
+
+static grpc_security_status ssl_create_security_connector(
+    grpc_channel_credentials *creds, grpc_call_credentials *call_creds,
+    const char *target, const grpc_channel_args *args,
+    grpc_channel_security_connector **sc, grpc_channel_args **new_args) {
+  grpc_ssl_credentials *c = (grpc_ssl_credentials *)creds;
+  grpc_security_status status = GRPC_SECURITY_OK;
+  size_t i = 0;
+  const char *overridden_target_name = NULL;
+  grpc_arg new_arg;
+
+  for (i = 0; args && i < args->num_args; i++) {
+    grpc_arg *arg = &args->args[i];
+    if (strcmp(arg->key, GRPC_SSL_TARGET_NAME_OVERRIDE_ARG) == 0 &&
+        arg->type == GRPC_ARG_STRING) {
+      overridden_target_name = arg->value.string;
+      break;
+    }
+  }
+  status = grpc_ssl_channel_security_connector_create(
+      call_creds, &c->config, target, overridden_target_name, sc);
+  if (status != GRPC_SECURITY_OK) {
+    return status;
+  }
+  new_arg.type = GRPC_ARG_STRING;
+  new_arg.key = GRPC_ARG_HTTP2_SCHEME;
+  new_arg.value.string = "https";
+  *new_args = grpc_channel_args_copy_and_add(args, &new_arg, 1);
+  return status;
+}
+
+static grpc_channel_credentials_vtable ssl_vtable = {
+    ssl_destruct, ssl_create_security_connector};
+
+static void ssl_build_config(const char *pem_root_certs,
+                             grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+                             grpc_ssl_config *config) {
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (pem_key_cert_pair != NULL) {
+    GPR_ASSERT(pem_key_cert_pair->private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pair->cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pair->private_key,
+                          &config->pem_private_key,
+                          &config->pem_private_key_size);
+    ssl_copy_key_material(pem_key_cert_pair->cert_chain,
+                          &config->pem_cert_chain,
+                          &config->pem_cert_chain_size);
+  }
+}
+
+grpc_channel_credentials *grpc_ssl_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pair,
+    void *reserved) {
+  grpc_ssl_credentials *c = gpr_malloc(sizeof(grpc_ssl_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_credentials_create(pem_root_certs=%s, "
+      "pem_key_cert_pair=%p, "
+      "reserved=%p)",
+      3, (pem_root_certs, pem_key_cert_pair, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  c->base.vtable = &ssl_vtable;
+  gpr_ref_init(&c->base.refcount, 1);
+  ssl_build_config(pem_root_certs, pem_key_cert_pair, &c->config);
+  return &c->base;
+}
+
+//
+// SSL Server Credentials.
+//
+
+static void ssl_server_destruct(grpc_server_credentials *creds) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  size_t i;
+  for (i = 0; i < c->config.num_key_cert_pairs; i++) {
+    if (c->config.pem_private_keys[i] != NULL) {
+      gpr_free(c->config.pem_private_keys[i]);
+    }
+    if (c->config.pem_cert_chains[i] != NULL) {
+      gpr_free(c->config.pem_cert_chains[i]);
+    }
+  }
+  if (c->config.pem_private_keys != NULL) gpr_free(c->config.pem_private_keys);
+  if (c->config.pem_private_keys_sizes != NULL) {
+    gpr_free(c->config.pem_private_keys_sizes);
+  }
+  if (c->config.pem_cert_chains != NULL) gpr_free(c->config.pem_cert_chains);
+  if (c->config.pem_cert_chains_sizes != NULL) {
+    gpr_free(c->config.pem_cert_chains_sizes);
+  }
+  if (c->config.pem_root_certs != NULL) gpr_free(c->config.pem_root_certs);
+}
+
+static grpc_security_status ssl_server_create_security_connector(
+    grpc_server_credentials *creds, grpc_server_security_connector **sc) {
+  grpc_ssl_server_credentials *c = (grpc_ssl_server_credentials *)creds;
+  return grpc_ssl_server_security_connector_create(&c->config, sc);
+}
+
+static grpc_server_credentials_vtable ssl_server_vtable = {
+    ssl_server_destruct, ssl_server_create_security_connector};
+
+static void ssl_build_server_config(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    grpc_ssl_server_config *config) {
+  size_t i;
+  config->client_certificate_request = client_certificate_request;
+  if (pem_root_certs != NULL) {
+    ssl_copy_key_material(pem_root_certs, &config->pem_root_certs,
+                          &config->pem_root_certs_size);
+  }
+  if (num_key_cert_pairs > 0) {
+    GPR_ASSERT(pem_key_cert_pairs != NULL);
+    config->pem_private_keys =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_cert_chains =
+        gpr_malloc(num_key_cert_pairs * sizeof(unsigned char *));
+    config->pem_private_keys_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+    config->pem_cert_chains_sizes =
+        gpr_malloc(num_key_cert_pairs * sizeof(size_t));
+  }
+  config->num_key_cert_pairs = num_key_cert_pairs;
+  for (i = 0; i < num_key_cert_pairs; i++) {
+    GPR_ASSERT(pem_key_cert_pairs[i].private_key != NULL);
+    GPR_ASSERT(pem_key_cert_pairs[i].cert_chain != NULL);
+    ssl_copy_key_material(pem_key_cert_pairs[i].private_key,
+                          &config->pem_private_keys[i],
+                          &config->pem_private_keys_sizes[i]);
+    ssl_copy_key_material(pem_key_cert_pairs[i].cert_chain,
+                          &config->pem_cert_chains[i],
+                          &config->pem_cert_chains_sizes[i]);
+  }
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs, int force_client_auth, void *reserved) {
+  return grpc_ssl_server_credentials_create_ex(
+      pem_root_certs, pem_key_cert_pairs, num_key_cert_pairs,
+      force_client_auth
+          ? GRPC_SSL_REQUEST_AND_REQUIRE_CLIENT_CERTIFICATE_AND_VERIFY
+          : GRPC_SSL_DONT_REQUEST_CLIENT_CERTIFICATE,
+      reserved);
+}
+
+grpc_server_credentials *grpc_ssl_server_credentials_create_ex(
+    const char *pem_root_certs, grpc_ssl_pem_key_cert_pair *pem_key_cert_pairs,
+    size_t num_key_cert_pairs,
+    grpc_ssl_client_certificate_request_type client_certificate_request,
+    void *reserved) {
+  grpc_ssl_server_credentials *c =
+      gpr_malloc(sizeof(grpc_ssl_server_credentials));
+  GRPC_API_TRACE(
+      "grpc_ssl_server_credentials_create_ex("
+      "pem_root_certs=%s, pem_key_cert_pairs=%p, num_key_cert_pairs=%lu, "
+      "client_certificate_request=%d, reserved=%p)",
+      5, (pem_root_certs, pem_key_cert_pairs, (unsigned long)num_key_cert_pairs,
+          client_certificate_request, reserved));
+  GPR_ASSERT(reserved == NULL);
+  memset(c, 0, sizeof(grpc_ssl_server_credentials));
+  c->base.type = GRPC_CHANNEL_CREDENTIALS_TYPE_SSL;
+  gpr_ref_init(&c->base.refcount, 1);
+  c->base.vtable = &ssl_server_vtable;
+  ssl_build_server_config(pem_root_certs, pem_key_cert_pairs,
+                          num_key_cert_pairs, client_certificate_request,
+                          &c->config);
+  return &c->base;
+}
diff --git a/src/core/lib/security/credentials/ssl/ssl_credentials.h b/src/core/lib/security/credentials/ssl/ssl_credentials.h
new file mode 100644
index 0000000000000000000000000000000000000000..f23dbdbe4941f262894724ffe9a9d323c1ec219f
--- /dev/null
+++ b/src/core/lib/security/credentials/ssl/ssl_credentials.h
@@ -0,0 +1,48 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#ifndef GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+#define GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H
+
+#include "src/core/lib/security/credentials/credentials.h"
+
+typedef struct {
+  grpc_channel_credentials base;
+  grpc_ssl_config config;
+} grpc_ssl_credentials;
+
+typedef struct {
+  grpc_server_credentials base;
+  grpc_ssl_server_config config;
+} grpc_ssl_server_credentials;
+
+#endif /* GRPC_CORE_LIB_SECURITY_CREDENTIALS_SSL_SSL_CREDENTIALS_H */
diff --git a/src/core/lib/security/auth_filters.h b/src/core/lib/security/transport/auth_filters.h
similarity index 90%
rename from src/core/lib/security/auth_filters.h
rename to src/core/lib/security/transport/auth_filters.h
index 7fb56c3f3afb27c0c38f8cb6d67e3d450b85a675..f688d4ed2183355944f37531fd0f7ef983de3724 100644
--- a/src/core/lib/security/auth_filters.h
+++ b/src/core/lib/security/transport/auth_filters.h
@@ -31,12 +31,12 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
-#define GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H
 
 #include "src/core/lib/channel/channel_stack.h"
 
 extern const grpc_channel_filter grpc_client_auth_filter;
 extern const grpc_channel_filter grpc_server_auth_filter;
 
-#endif /* GRPC_CORE_LIB_SECURITY_AUTH_FILTERS_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_AUTH_FILTERS_H */
diff --git a/src/core/lib/security/client_auth_filter.c b/src/core/lib/security/transport/client_auth_filter.c
similarity index 98%
rename from src/core/lib/security/client_auth_filter.c
rename to src/core/lib/security/transport/client_auth_filter.c
index f8089739094959dd9cd7722b9c6e7cc1bd826f51..27208ebb605f4081074ae279068cf66d0e9c3a33 100644
--- a/src/core/lib/security/client_auth_filter.c
+++ b/src/core/lib/security/transport/client_auth_filter.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/auth_filters.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <string.h>
 
@@ -40,9 +40,9 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_stack.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/surface/call.h"
 #include "src/core/lib/transport/static_metadata.h"
diff --git a/src/core/lib/security/handshake.c b/src/core/lib/security/transport/handshake.c
similarity index 98%
rename from src/core/lib/security/handshake.c
rename to src/core/lib/security/transport/handshake.c
index d5fe0c7b7d93560284ed82585205a0c4e1d82f37..6561f4b47d5c8801ec42bf1974a18b317922f737 100644
--- a/src/core/lib/security/handshake.c
+++ b/src/core/lib/security/transport/handshake.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/handshake.h"
+#include "src/core/lib/security/transport/handshake.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -39,8 +39,8 @@
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice_buffer.h>
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 
 #define GRPC_INITIAL_HANDSHAKE_BUFFER_SIZE 256
 
diff --git a/src/core/lib/security/handshake.h b/src/core/lib/security/transport/handshake.h
similarity index 90%
rename from src/core/lib/security/handshake.h
rename to src/core/lib/security/transport/handshake.h
index f34476ed49d777d65d55c6ff0ead98d37d4120a3..6ed850b31530ca41410a554f668ef279e307ff9d 100644
--- a/src/core/lib/security/handshake.h
+++ b/src/core/lib/security/transport/handshake.h
@@ -31,11 +31,11 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
-#define GRPC_CORE_LIB_SECURITY_HANDSHAKE_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H
 
 #include "src/core/lib/iomgr/endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 /* Calls the callback upon completion. Takes owership of handshaker. */
 void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
@@ -48,4 +48,4 @@ void grpc_do_security_handshake(grpc_exec_ctx *exec_ctx,
 
 void grpc_security_handshake_shutdown(grpc_exec_ctx *exec_ctx, void *handshake);
 
-#endif /* GRPC_CORE_LIB_SECURITY_HANDSHAKE_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_HANDSHAKE_H */
diff --git a/src/core/lib/security/secure_endpoint.c b/src/core/lib/security/transport/secure_endpoint.c
similarity index 99%
rename from src/core/lib/security/secure_endpoint.c
rename to src/core/lib/security/transport/secure_endpoint.c
index 27b0e989107bcf938f537629e36d808dae0ce2a3..4438c8e559602c85c074a0c114bdab419b42915c 100644
--- a/src/core/lib/security/secure_endpoint.c
+++ b/src/core/lib/security/transport/secure_endpoint.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/slice.h>
diff --git a/src/core/lib/security/secure_endpoint.h b/src/core/lib/security/transport/secure_endpoint.h
similarity index 91%
rename from src/core/lib/security/secure_endpoint.h
rename to src/core/lib/security/transport/secure_endpoint.h
index ff1c6639de6f185199cc9f4f6982037fdba5b0d9..d00075b769277dd806ab3a5aad599e64f09fc613 100644
--- a/src/core/lib/security/secure_endpoint.h
+++ b/src/core/lib/security/transport/secure_endpoint.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
-#define GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H
 
 #include <grpc/support/slice.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -46,4 +46,4 @@ grpc_endpoint *grpc_secure_endpoint_create(
     struct tsi_frame_protector *protector, grpc_endpoint *to_wrap,
     gpr_slice *leftover_slices, size_t leftover_nslices);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURE_ENDPOINT_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURE_ENDPOINT_H */
diff --git a/src/core/lib/security/security_connector.c b/src/core/lib/security/transport/security_connector.c
similarity index 99%
rename from src/core/lib/security/security_connector.c
rename to src/core/lib/security/transport/security_connector.c
index 2d2023bdf5b942077c0208ad324fc6bfaf1b7df4..72173e7c9dc7edf735e2c4ec95031b667133a1cd 100644
--- a/src/core/lib/security/security_connector.c
+++ b/src/core/lib/security/transport/security_connector.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/transport/security_connector.h"
 
 #include <stdbool.h>
 #include <string.h>
@@ -43,10 +43,10 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/transport/chttp2/alpn/alpn.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/handshake.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/handshake.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/load_file.h"
 #include "src/core/lib/support/string.h"
diff --git a/src/core/lib/security/security_connector.h b/src/core/lib/security/transport/security_connector.h
similarity index 98%
rename from src/core/lib/security/security_connector.h
rename to src/core/lib/security/transport/security_connector.h
index 2c893cd5e99f73fc606d63d1c4b259db912a05e3..84e586deaa01baf6e59ed9001dec50b49c8d9071 100644
--- a/src/core/lib/security/security_connector.h
+++ b/src/core/lib/security/transport/security_connector.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
-#define GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H
+#ifndef GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
+#define GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H
 
 #include <grpc/grpc_security.h>
 #include "src/core/lib/iomgr/endpoint.h"
@@ -263,4 +263,4 @@ tsi_peer tsi_shallow_peer_from_ssl_auth_context(
     const grpc_auth_context *auth_context);
 void tsi_shallow_peer_destruct(tsi_peer *peer);
 
-#endif /* GRPC_CORE_LIB_SECURITY_SECURITY_CONNECTOR_H */
+#endif /* GRPC_CORE_LIB_SECURITY_TRANSPORT_SECURITY_CONNECTOR_H */
diff --git a/src/core/lib/security/server_auth_filter.c b/src/core/lib/security/transport/server_auth_filter.c
similarity index 98%
rename from src/core/lib/security/server_auth_filter.c
rename to src/core/lib/security/transport/server_auth_filter.c
index 0290f8a5b5c2a02ae8c240a30442d1eef1858fb6..714e0adfc9632db553b779e56927372ddb86a4e6 100644
--- a/src/core/lib/security/server_auth_filter.c
+++ b/src/core/lib/security/transport/server_auth_filter.c
@@ -33,9 +33,9 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
 
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
diff --git a/src/core/lib/security/b64.c b/src/core/lib/security/util/b64.c
similarity index 99%
rename from src/core/lib/security/b64.c
rename to src/core/lib/security/util/b64.c
index 87f0e05280e5ba9b0776706c2d5ce3bdc72677d4..9da42e4e73449c860fcefd685d54fc92d6c7215b 100644
--- a/src/core/lib/security/b64.c
+++ b/src/core/lib/security/util/b64.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 
 #include <stdint.h>
 #include <string.h>
diff --git a/src/core/lib/security/b64.h b/src/core/lib/security/util/b64.h
similarity index 94%
rename from src/core/lib/security/b64.h
rename to src/core/lib/security/util/b64.h
index c515e7af2c99adab6b92e21d275f5e3d30d50e84..69080952873dc7800376f814a1de7c866cdfa694 100644
--- a/src/core/lib/security/b64.h
+++ b/src/core/lib/security/util/b64.h
@@ -31,8 +31,8 @@
  *
  */
 
-#ifndef GRPC_CORE_LIB_SECURITY_B64_H
-#define GRPC_CORE_LIB_SECURITY_B64_H
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_B64_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_B64_H
 
 #include <grpc/support/slice.h>
 
@@ -49,4 +49,4 @@ gpr_slice grpc_base64_decode(const char *b64, int url_safe);
 gpr_slice grpc_base64_decode_with_len(const char *b64, size_t b64_len,
                                       int url_safe);
 
-#endif /* GRPC_CORE_LIB_SECURITY_B64_H */
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_B64_H */
diff --git a/test/cpp/qps/sync_unary_ping_pong_test.cc b/src/core/lib/security/util/json_util.c
similarity index 60%
rename from test/cpp/qps/sync_unary_ping_pong_test.cc
rename to src/core/lib/security/util/json_util.c
index aa0c0c3013900cdc475a4cb82f49820c29f80d58..7eed039baa0871bbddc64395fe486f02528a2104 100644
--- a/test/cpp/qps/sync_unary_ping_pong_test.cc
+++ b/src/core/lib/security/util/json_util.c
@@ -31,47 +31,31 @@
  *
  */
 
-#include <set>
+#include "src/core/lib/security/util/json_util.h"
 
-#include <grpc/support/log.h>
-
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
-
-namespace grpc {
-namespace testing {
-
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
-
-static void RunSynchronousUnaryPingPong() {
-  gpr_log(GPR_INFO, "Running Synchronous Unary Ping Pong");
-
-  ClientConfig client_config;
-  client_config.set_client_type(SYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1);
-  client_config.set_client_channels(1);
-  client_config.set_rpc_type(UNARY);
-  client_config.mutable_load_params()->mutable_closed_loop();
+#include <string.h>
 
-  ServerConfig server_config;
-  server_config.set_server_type(SYNC_SERVER);
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPS(*result);
-  GetReporter()->ReportLatency(*result);
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name) {
+  grpc_json *child;
+  for (child = json->child; child != NULL; child = child->next) {
+    if (strcmp(child->key, prop_name) == 0) break;
+  }
+  if (child == NULL || child->type != GRPC_JSON_STRING) {
+    gpr_log(GPR_ERROR, "Invalid or missing %s property.", prop_name);
+    return NULL;
+  }
+  return child->value;
 }
 
-}  // namespace testing
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
-
-  grpc::testing::RunSynchronousUnaryPingPong();
-
-  return 0;
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name,
+                                    char **copied_value) {
+  const char *prop_value = grpc_json_get_string_property(json, prop_name);
+  if (prop_value == NULL) return false;
+  *copied_value = gpr_strdup(prop_value);
+  return true;
 }
diff --git a/test/cpp/qps/sync_streaming_ping_pong_test.cc b/src/core/lib/security/util/json_util.h
similarity index 59%
rename from test/cpp/qps/sync_streaming_ping_pong_test.cc
rename to src/core/lib/security/util/json_util.h
index 67c62f4baea6f3602dd9b97e09be22fb6038c4b3..137900593f697e3ee46bd4608de3756f67a51c6b 100644
--- a/test/cpp/qps/sync_streaming_ping_pong_test.cc
+++ b/src/core/lib/security/util/json_util.h
@@ -31,46 +31,25 @@
  *
  */
 
-#include <set>
+#ifndef GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
+#define GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H
 
-#include <grpc/support/log.h>
+#include <stdbool.h>
 
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
+#include "src/core/lib/json/json.h"
 
-namespace grpc {
-namespace testing {
+// Constants.
+#define GRPC_AUTH_JSON_TYPE_INVALID "invalid"
+#define GRPC_AUTH_JSON_TYPE_SERVICE_ACCOUNT "service_account"
+#define GRPC_AUTH_JSON_TYPE_AUTHORIZED_USER "authorized_user"
 
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
+// Gets a child property from a json node.
+const char *grpc_json_get_string_property(const grpc_json *json,
+                                          const char *prop_name);
 
-static void RunSynchronousStreamingPingPong() {
-  gpr_log(GPR_INFO, "Running Synchronous Streaming Ping Pong");
+// Copies the value of the json child property specified by prop_name.
+// Returns false if the property was not found.
+bool grpc_copy_json_string_property(const grpc_json *json,
+                                    const char *prop_name, char **copied_value);
 
-  ClientConfig client_config;
-  client_config.set_client_type(SYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1);
-  client_config.set_client_channels(1);
-  client_config.set_rpc_type(STREAMING);
-  client_config.mutable_load_params()->mutable_closed_loop();
-
-  ServerConfig server_config;
-  server_config.set_server_type(SYNC_SERVER);
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPS(*result);
-  GetReporter()->ReportLatency(*result);
-}
-}  // namespace testing
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
-
-  grpc::testing::RunSynchronousStreamingPingPong();
-
-  return 0;
-}
+#endif /* GRPC_CORE_LIB_SECURITY_UTIL_JSON_UTIL_H */
diff --git a/src/core/lib/support/murmur_hash.c b/src/core/lib/support/murmur_hash.c
index 5711fff0c0fc207257c00bf60af51f1c2d4c1be1..7137c1f31332c3ea2eff64840be341849c2a6b29 100644
--- a/src/core/lib/support/murmur_hash.c
+++ b/src/core/lib/support/murmur_hash.c
@@ -33,6 +33,8 @@
 
 #include "src/core/lib/support/murmur_hash.h"
 
+#include <string.h>
+
 #define ROTL32(x, r) ((x) << (r)) | ((x) >> (32 - (r)))
 
 #define FMIX32(h)    \
@@ -42,10 +44,6 @@
   (h) *= 0xc2b2ae35; \
   (h) ^= (h) >> 16;
 
-/* Block read - if your platform needs to do endian-swapping or can only
-   handle aligned reads, do the conversion here */
-#define GETBLOCK32(p, i) (p)[(i)]
-
 uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
   const uint8_t *data = (const uint8_t *)key;
   const size_t nblocks = len / 4;
@@ -62,7 +60,7 @@ uint32_t gpr_murmur_hash3(const void *key, size_t len, uint32_t seed) {
 
   /* body */
   for (i = -(int)nblocks; i; i++) {
-    k1 = GETBLOCK32(blocks, i);
+    memcpy(&k1, blocks + i, sizeof(uint32_t));
 
     k1 *= c1;
     k1 = ROTL32(k1, 15);
diff --git a/src/core/lib/surface/call.c b/src/core/lib/surface/call.c
index 370360b8f60494406679d7a5437e94da5abdea80..74a09cf6edf748b987592a713258a1469ad7bd37 100644
--- a/src/core/lib/surface/call.c
+++ b/src/core/lib/surface/call.c
@@ -65,12 +65,6 @@
       - status/close recv (depending on client/server) */
 #define MAX_CONCURRENT_BATCHES 6
 
-typedef struct {
-  grpc_ioreq_completion_func on_complete;
-  void *user_data;
-  int success;
-} completed_request;
-
 #define MAX_SEND_EXTRA_METADATA_COUNT 3
 
 /* Status data for a request can come from several sources; this
@@ -97,25 +91,6 @@ typedef struct {
   grpc_mdstr *details;
 } received_status;
 
-/* How far through the GRPC stream have we read? */
-typedef enum {
-  /* We are still waiting for initial metadata to complete */
-  READ_STATE_INITIAL = 0,
-  /* We have gotten initial metadata, and are reading either
-     messages or trailing metadata */
-  READ_STATE_GOT_INITIAL_METADATA,
-  /* The stream is closed for reading */
-  READ_STATE_READ_CLOSED,
-  /* The stream is closed for reading & writing */
-  READ_STATE_STREAM_CLOSED
-} read_state;
-
-typedef enum {
-  WRITE_STATE_INITIAL = 0,
-  WRITE_STATE_STARTED,
-  WRITE_STATE_WRITE_CLOSED
-} write_state;
-
 typedef struct batch_control {
   grpc_call *call;
   grpc_cq_completion cq_completion;
diff --git a/src/core/lib/surface/completion_queue.c b/src/core/lib/surface/completion_queue.c
index 1f82c3bad2e679165927c1ab59728231c737937a..5eb7cf1bf4f4dd7cb2de8579ce29b0be56428b4d 100644
--- a/src/core/lib/surface/completion_queue.c
+++ b/src/core/lib/surface/completion_queue.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -70,6 +70,8 @@ struct grpc_completion_queue {
   int shutdown;
   int shutdown_called;
   int is_server_cq;
+  /** Can the server cq accept incoming channels */
+  int is_non_listening_server_cq;
   int num_pluckers;
   plucker pluckers[GRPC_MAX_COMPLETION_QUEUE_PLUCKERS];
   grpc_closure pollset_shutdown_done;
@@ -84,6 +86,7 @@ struct grpc_completion_queue {
 };
 
 #define POLLSET_FROM_CQ(cq) ((grpc_pollset *)(cq + 1))
+#define CQ_FROM_POLLSET(ps) (((grpc_completion_queue *)ps) - 1)
 
 static gpr_mu g_freelist_mu;
 static grpc_completion_queue *g_freelist;
@@ -149,6 +152,7 @@ grpc_completion_queue *grpc_completion_queue_create(void *reserved) {
   cc->shutdown = 0;
   cc->shutdown_called = 0;
   cc->is_server_cq = 0;
+  cc->is_non_listening_server_cq = 0;
   cc->num_pluckers = 0;
 #ifndef NDEBUG
   cc->outstanding_tag_count = 0;
@@ -511,6 +515,18 @@ grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc) {
   return POLLSET_FROM_CQ(cc);
 }
 
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps) {
+  return CQ_FROM_POLLSET(ps);
+}
+
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc) {
+  cc->is_non_listening_server_cq = 1;
+}
+
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc) {
+  return (cc->is_non_listening_server_cq == 1);
+}
+
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc) { cc->is_server_cq = 1; }
 
 int grpc_cq_is_server_cq(grpc_completion_queue *cc) { return cc->is_server_cq; }
diff --git a/src/core/lib/surface/completion_queue.h b/src/core/lib/surface/completion_queue.h
index eef82cf01482624ab9a2c943b72c637f288fccf6..3d0dd13c53b881cfc2bde0293f6fad0a36acee54 100644
--- a/src/core/lib/surface/completion_queue.h
+++ b/src/core/lib/surface/completion_queue.h
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -81,7 +81,10 @@ void grpc_cq_end_op(grpc_exec_ctx *exec_ctx, grpc_completion_queue *cc,
                     void *done_arg, grpc_cq_completion *storage);
 
 grpc_pollset *grpc_cq_pollset(grpc_completion_queue *cc);
+grpc_completion_queue *grpc_cq_from_pollset(grpc_pollset *ps);
 
+void grpc_cq_mark_non_listening_server_cq(grpc_completion_queue *cc);
+bool grpc_cq_is_non_listening_server_cq(grpc_completion_queue *cc);
 void grpc_cq_mark_server_cq(grpc_completion_queue *cc);
 int grpc_cq_is_server_cq(grpc_completion_queue *cc);
 
diff --git a/src/core/lib/surface/init_secure.c b/src/core/lib/surface/init_secure.c
index 3fda2c9e1e45e6305338cf98fa6a317a3445d818..7ee7b51568ede51b16913f0564cbc0e7a72b64eb 100644
--- a/src/core/lib/surface/init_secure.c
+++ b/src/core/lib/surface/init_secure.c
@@ -37,10 +37,10 @@
 #include <string.h>
 
 #include "src/core/lib/debug/trace.h"
-#include "src/core/lib/security/auth_filters.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/secure_endpoint.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/transport/auth_filters.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/channel_init.h"
 #include "src/core/lib/tsi/transport_security_interface.h"
 
diff --git a/src/core/lib/surface/server.c b/src/core/lib/surface/server.c
index 3c8fa3aadc8c58f9487a1e8954078a257bcf97c9..dab7e35748c6156d377d87f8ac1bbea9ca94e108 100644
--- a/src/core/lib/surface/server.c
+++ b/src/core/lib/surface/server.c
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -69,11 +69,6 @@ typedef struct call_data call_data;
 typedef struct channel_data channel_data;
 typedef struct registered_method registered_method;
 
-typedef struct {
-  call_data *next;
-  call_data *prev;
-} call_link;
-
 typedef enum { BATCH_CALL, REGISTERED_CALL } requested_call_type;
 
 typedef struct requested_call {
@@ -81,7 +76,6 @@ typedef struct requested_call {
   void *tag;
   grpc_server *server;
   grpc_completion_queue *cq_bound_to_call;
-  grpc_completion_queue *cq_for_notification;
   grpc_call **call;
   grpc_cq_completion completion;
   grpc_metadata_array *initial_metadata;
@@ -108,6 +102,7 @@ struct channel_data {
   grpc_server *server;
   grpc_connectivity_state connectivity_state;
   grpc_channel *channel;
+  size_t cq_idx;
   /* linked list of all channels on a server */
   channel_data *next;
   channel_data *prev;
@@ -172,7 +167,7 @@ struct request_matcher {
   grpc_server *server;
   call_data *pending_head;
   call_data *pending_tail;
-  gpr_stack_lockfree *requests;
+  gpr_stack_lockfree **requests_per_cq;
 };
 
 struct registered_method {
@@ -180,6 +175,7 @@ struct registered_method {
   char *host;
   grpc_server_register_method_payload_handling payload_handling;
   uint32_t flags;
+  /* one request matcher per method */
   request_matcher request_matcher;
   registered_method *next;
 };
@@ -195,6 +191,7 @@ struct grpc_server {
   grpc_completion_queue **cqs;
   grpc_pollset **pollsets;
   size_t cq_count;
+  bool started;
 
   /* The two following mutexes control access to server-state
      mu_global controls access to non-call-related state (e.g., channel state)
@@ -207,6 +204,7 @@ struct grpc_server {
   gpr_mu mu_call;   /* mutex for call-specific state */
 
   registered_method *registered_methods;
+  /** one request matcher for unregistered methods */
   request_matcher unregistered_request_matcher;
   /** free list of available requested_calls indices */
   gpr_stack_lockfree *request_freelist;
@@ -234,7 +232,7 @@ struct grpc_server {
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *calld, bool success);
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc);
+                      size_t cq_idx, requested_call *rc);
 /* Before calling maybe_finish_shutdown, we must hold mu_global and not
    hold mu_call */
 static void maybe_finish_shutdown(grpc_exec_ctx *exec_ctx, grpc_server *server);
@@ -312,12 +310,19 @@ static void request_matcher_init(request_matcher *rm, size_t entries,
                                  grpc_server *server) {
   memset(rm, 0, sizeof(*rm));
   rm->server = server;
-  rm->requests = gpr_stack_lockfree_create(entries);
+  rm->requests_per_cq =
+      gpr_malloc(sizeof(*rm->requests_per_cq) * server->cq_count);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    rm->requests_per_cq[i] = gpr_stack_lockfree_create(entries);
+  }
 }
 
 static void request_matcher_destroy(request_matcher *rm) {
-  GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests) == -1);
-  gpr_stack_lockfree_destroy(rm->requests);
+  for (size_t i = 0; i < rm->server->cq_count; i++) {
+    GPR_ASSERT(gpr_stack_lockfree_pop(rm->requests_per_cq[i]) == -1);
+    gpr_stack_lockfree_destroy(rm->requests_per_cq[i]);
+  }
+  gpr_free(rm->requests_per_cq);
 }
 
 static void kill_zombie(grpc_exec_ctx *exec_ctx, void *elem, bool success) {
@@ -343,8 +348,11 @@ static void request_matcher_kill_requests(grpc_exec_ctx *exec_ctx,
                                           grpc_server *server,
                                           request_matcher *rm) {
   int request_id;
-  while ((request_id = gpr_stack_lockfree_pop(rm->requests)) != -1) {
-    fail_call(exec_ctx, server, &server->requested_calls[request_id]);
+  for (size_t i = 0; i < server->cq_count; i++) {
+    while ((request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[i])) !=
+           -1) {
+      fail_call(exec_ctx, server, i, &server->requested_calls[request_id]);
+    }
   }
 }
 
@@ -364,15 +372,19 @@ static void server_delete(grpc_exec_ctx *exec_ctx, grpc_server *server) {
   gpr_mu_destroy(&server->mu_call);
   while ((rm = server->registered_methods) != NULL) {
     server->registered_methods = rm->next;
-    request_matcher_destroy(&rm->request_matcher);
+    if (server->started) {
+      request_matcher_destroy(&rm->request_matcher);
+    }
     gpr_free(rm->method);
     gpr_free(rm->host);
     gpr_free(rm);
   }
+  if (server->started) {
+    request_matcher_destroy(&server->unregistered_request_matcher);
+  }
   for (i = 0; i < server->cq_count; i++) {
     GRPC_CQ_INTERNAL_UNREF(server->cqs[i], "server");
   }
-  request_matcher_destroy(&server->unregistered_request_matcher);
   gpr_stack_lockfree_destroy(server->request_freelist);
   gpr_free(server->cqs);
   gpr_free(server->pollsets);
@@ -453,11 +465,11 @@ static void done_request_event(grpc_exec_ctx *exec_ctx, void *req,
 }
 
 static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                         call_data *calld, requested_call *rc) {
+                         call_data *calld, size_t cq_idx, requested_call *rc) {
   grpc_call_set_completion_queue(exec_ctx, calld->call, rc->cq_bound_to_call);
   grpc_call *call = calld->call;
   *rc->call = call;
-  calld->cq_new = rc->cq_for_notification;
+  calld->cq_new = server->cqs[cq_idx];
   GPR_SWAP(grpc_metadata_array, *rc->initial_metadata, calld->initial_metadata);
   switch (rc->type) {
     case BATCH_CALL:
@@ -492,7 +504,9 @@ static void publish_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
 }
 
 static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
-  call_data *calld = arg;
+  grpc_call_element *call_elem = arg;
+  call_data *calld = call_elem->call_data;
+  channel_data *chand = call_elem->channel_data;
   request_matcher *rm = calld->request_matcher;
   grpc_server *server = rm->server;
 
@@ -507,26 +521,34 @@ static void publish_new_rpc(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
     return;
   }
 
-  int request_id = gpr_stack_lockfree_pop(rm->requests);
-  if (request_id == -1) {
-    gpr_mu_lock(&server->mu_call);
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = PENDING;
-    gpr_mu_unlock(&calld->mu_state);
-    if (rm->pending_head == NULL) {
-      rm->pending_tail = rm->pending_head = calld;
+  for (size_t i = 0; i < server->cq_count; i++) {
+    size_t cq_idx = (chand->cq_idx + i) % server->cq_count;
+    int request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
+    if (request_id == -1) {
+      continue;
     } else {
-      rm->pending_tail->pending_next = calld;
-      rm->pending_tail = calld;
+      gpr_mu_lock(&calld->mu_state);
+      calld->state = ACTIVATED;
+      gpr_mu_unlock(&calld->mu_state);
+      publish_call(exec_ctx, server, calld, cq_idx,
+                   &server->requested_calls[request_id]);
+      return; /* early out */
     }
-    calld->pending_next = NULL;
-    gpr_mu_unlock(&server->mu_call);
+  }
+
+  /* no cq to take the request found: queue it on the slow list */
+  gpr_mu_lock(&server->mu_call);
+  gpr_mu_lock(&calld->mu_state);
+  calld->state = PENDING;
+  gpr_mu_unlock(&calld->mu_state);
+  if (rm->pending_head == NULL) {
+    rm->pending_tail = rm->pending_head = calld;
   } else {
-    gpr_mu_lock(&calld->mu_state);
-    calld->state = ACTIVATED;
-    gpr_mu_unlock(&calld->mu_state);
-    publish_call(exec_ctx, server, calld, &server->requested_calls[request_id]);
+    rm->pending_tail->pending_next = calld;
+    rm->pending_tail = calld;
   }
+  calld->pending_next = NULL;
+  gpr_mu_unlock(&server->mu_call);
 }
 
 static void finish_start_new_rpc(
@@ -548,14 +570,14 @@ static void finish_start_new_rpc(
 
   switch (payload_handling) {
     case GRPC_SRM_PAYLOAD_NONE:
-      publish_new_rpc(exec_ctx, calld, true);
+      publish_new_rpc(exec_ctx, elem, true);
       break;
     case GRPC_SRM_PAYLOAD_READ_INITIAL_BYTE_BUFFER: {
       grpc_op op;
       memset(&op, 0, sizeof(op));
       op.op = GRPC_OP_RECV_MESSAGE;
       op.data.recv_message = &calld->payload;
-      grpc_closure_init(&calld->publish, publish_new_rpc, calld);
+      grpc_closure_init(&calld->publish, publish_new_rpc, elem);
       grpc_call_start_batch_and_execute(exec_ctx, calld->call, &op, 1,
                                         &calld->publish);
       break;
@@ -637,14 +659,16 @@ static int num_channels(grpc_server *server) {
 
 static void kill_pending_work_locked(grpc_exec_ctx *exec_ctx,
                                      grpc_server *server) {
-  registered_method *rm;
-  request_matcher_kill_requests(exec_ctx, server,
-                                &server->unregistered_request_matcher);
-  request_matcher_zombify_all_pending_calls(
-      exec_ctx, &server->unregistered_request_matcher);
-  for (rm = server->registered_methods; rm; rm = rm->next) {
-    request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
-    request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+  if (server->started) {
+    request_matcher_kill_requests(exec_ctx, server,
+                                  &server->unregistered_request_matcher);
+    request_matcher_zombify_all_pending_calls(
+        exec_ctx, &server->unregistered_request_matcher);
+    for (registered_method *rm = server->registered_methods; rm;
+         rm = rm->next) {
+      request_matcher_kill_requests(exec_ctx, server, &rm->request_matcher);
+      request_matcher_zombify_all_pending_calls(exec_ctx, &rm->request_matcher);
+    }
   }
 }
 
@@ -895,25 +919,46 @@ const grpc_channel_filter grpc_server_top_filter = {
     "server",
 };
 
-void grpc_server_register_completion_queue(grpc_server *server,
-                                           grpc_completion_queue *cq,
-                                           void *reserved) {
+static void register_completion_queue(grpc_server *server,
+                                      grpc_completion_queue *cq,
+                                      bool is_non_listening, void *reserved) {
   size_t i, n;
-  GRPC_API_TRACE(
-      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
-      (server, cq, reserved));
   GPR_ASSERT(!reserved);
   for (i = 0; i < server->cq_count; i++) {
     if (server->cqs[i] == cq) return;
   }
-  GRPC_CQ_INTERNAL_REF(cq, "server");
+
   grpc_cq_mark_server_cq(cq);
+
+  if (is_non_listening) {
+    grpc_cq_mark_non_listening_server_cq(cq);
+  }
+
+  GRPC_CQ_INTERNAL_REF(cq, "server");
   n = server->cq_count++;
   server->cqs = gpr_realloc(server->cqs,
                             server->cq_count * sizeof(grpc_completion_queue *));
   server->cqs[n] = cq;
 }
 
+void grpc_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq,
+                                           void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_completion_queue(server=%p, cq=%p, reserved=%p)", 3,
+      (server, cq, reserved));
+  register_completion_queue(server, cq, false, reserved);
+}
+
+void grpc_server_register_non_listening_completion_queue(
+    grpc_server *server, grpc_completion_queue *cq, void *reserved) {
+  GRPC_API_TRACE(
+      "grpc_server_register_non_listening_completion_queue(server=%p, cq=%p, "
+      "reserved=%p)",
+      3, (server, cq, reserved));
+  register_completion_queue(server, cq, true, reserved);
+}
+
 grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   size_t i;
 
@@ -940,8 +985,6 @@ grpc_server *grpc_server_create(const grpc_channel_args *args, void *reserved) {
   for (i = 0; i < (size_t)server->max_requested_calls; i++) {
     gpr_stack_lockfree_push(server->request_freelist, (int)i);
   }
-  request_matcher_init(&server->unregistered_request_matcher,
-                       server->max_requested_calls, server);
   server->requested_calls = gpr_malloc(server->max_requested_calls *
                                        sizeof(*server->requested_calls));
 
@@ -985,8 +1028,6 @@ void *grpc_server_register_method(
   }
   m = gpr_malloc(sizeof(registered_method));
   memset(m, 0, sizeof(*m));
-  request_matcher_init(&m->request_matcher, server->max_requested_calls,
-                       server);
   m->method = gpr_strdup(method);
   m->host = gpr_strdup(host);
   m->next = server->registered_methods;
@@ -1003,13 +1044,23 @@ void grpc_server_start(grpc_server *server) {
 
   GRPC_API_TRACE("grpc_server_start(server=%p)", 1, (server));
 
+  server->started = true;
+  size_t pollset_count = 0;
   server->pollsets = gpr_malloc(sizeof(grpc_pollset *) * server->cq_count);
   for (i = 0; i < server->cq_count; i++) {
-    server->pollsets[i] = grpc_cq_pollset(server->cqs[i]);
+    if (!grpc_cq_is_non_listening_server_cq(server->cqs[i])) {
+      server->pollsets[pollset_count++] = grpc_cq_pollset(server->cqs[i]);
+    }
+  }
+  request_matcher_init(&server->unregistered_request_matcher,
+                       server->max_requested_calls, server);
+  for (registered_method *rm = server->registered_methods; rm; rm = rm->next) {
+    request_matcher_init(&rm->request_matcher, server->max_requested_calls,
+                         server);
   }
 
   for (l = server->listeners; l; l = l->next) {
-    l->start(&exec_ctx, server, l->arg, server->pollsets, server->cq_count);
+    l->start(&exec_ctx, server, l->arg, server->pollsets, pollset_count);
   }
 
   grpc_exec_ctx_finish(&exec_ctx);
@@ -1017,8 +1068,8 @@ void grpc_server_start(grpc_server *server) {
 
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args) {
-  size_t i;
   size_t num_registered_methods;
   size_t alloc;
   registered_method *rm;
@@ -1033,12 +1084,6 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   uint32_t max_probes = 0;
   grpc_transport_op op;
 
-  for (i = 0; i < s->cq_count; i++) {
-    memset(&op, 0, sizeof(op));
-    op.bind_pollset = grpc_cq_pollset(s->cqs[i]);
-    grpc_transport_perform_op(exec_ctx, transport, &op);
-  }
-
   channel =
       grpc_channel_create(exec_ctx, NULL, args, GRPC_SERVER_CHANNEL, transport);
   chand = (channel_data *)grpc_channel_stack_element(
@@ -1048,6 +1093,17 @@ void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *s,
   server_ref(s);
   chand->channel = channel;
 
+  size_t cq_idx;
+  grpc_completion_queue *accepting_cq = grpc_cq_from_pollset(accepting_pollset);
+  for (cq_idx = 0; cq_idx < s->cq_count; cq_idx++) {
+    if (s->cqs[cq_idx] == accepting_cq) break;
+  }
+  if (cq_idx == s->cq_count) {
+    /* completion queue not found: pick a random one to publish new calls to */
+    cq_idx = (size_t)rand() % s->cq_count;
+  }
+  chand->cq_idx = cq_idx;
+
   num_registered_methods = 0;
   for (rm = s->registered_methods; rm; rm = rm->next) {
     num_registered_methods++;
@@ -1218,19 +1274,19 @@ void grpc_server_add_listener(
 }
 
 static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
-                                          grpc_server *server,
+                                          grpc_server *server, size_t cq_idx,
                                           requested_call *rc) {
   call_data *calld = NULL;
   request_matcher *rm = NULL;
   int request_id;
   if (gpr_atm_acq_load(&server->shutdown_flag)) {
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   request_id = gpr_stack_lockfree_pop(server->request_freelist);
   if (request_id == -1) {
     /* out of request ids: just fail this one */
-    fail_call(exec_ctx, server, rc);
+    fail_call(exec_ctx, server, cq_idx, rc);
     return GRPC_CALL_OK;
   }
   switch (rc->type) {
@@ -1243,12 +1299,12 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
   }
   server->requested_calls[request_id] = *rc;
   gpr_free(rc);
-  if (gpr_stack_lockfree_push(rm->requests, request_id)) {
+  if (gpr_stack_lockfree_push(rm->requests_per_cq[cq_idx], request_id)) {
     /* this was the first queued request: we need to lock and start
        matching calls */
     gpr_mu_lock(&server->mu_call);
     while ((calld = rm->pending_head) != NULL) {
-      request_id = gpr_stack_lockfree_pop(rm->requests);
+      request_id = gpr_stack_lockfree_pop(rm->requests_per_cq[cq_idx]);
       if (request_id == -1) break;
       rm->pending_head = calld->pending_next;
       gpr_mu_unlock(&server->mu_call);
@@ -1264,7 +1320,7 @@ static grpc_call_error queue_call_request(grpc_exec_ctx *exec_ctx,
         GPR_ASSERT(calld->state == PENDING);
         calld->state = ACTIVATED;
         gpr_mu_unlock(&calld->mu_state);
-        publish_call(exec_ctx, server, calld,
+        publish_call(exec_ctx, server, calld, cq_idx,
                      &server->requested_calls[request_id]);
       }
       gpr_mu_lock(&server->mu_call);
@@ -1288,7 +1344,13 @@ grpc_call_error grpc_server_request_call(
       "cq_bound_to_call=%p, cq_for_notification=%p, tag=%p)",
       7, (server, call, details, initial_metadata, cq_bound_to_call,
           cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1299,11 +1361,10 @@ grpc_call_error grpc_server_request_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.batch.details = details;
   rc->initial_metadata = initial_metadata;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
@@ -1325,7 +1386,14 @@ grpc_call_error grpc_server_request_registered_call(
       "tag=%p)",
       9, (server, rmp, call, deadline, initial_metadata, optional_payload,
           cq_bound_to_call, cq_for_notification, tag));
-  if (!grpc_cq_is_server_cq(cq_for_notification)) {
+
+  size_t cq_idx;
+  for (cq_idx = 0; cq_idx < server->cq_count; cq_idx++) {
+    if (server->cqs[cq_idx] == cq_for_notification) {
+      break;
+    }
+  }
+  if (cq_idx == server->cq_count) {
     gpr_free(rc);
     error = GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE;
     goto done;
@@ -1341,26 +1409,25 @@ grpc_call_error grpc_server_request_registered_call(
   rc->server = server;
   rc->tag = tag;
   rc->cq_bound_to_call = cq_bound_to_call;
-  rc->cq_for_notification = cq_for_notification;
   rc->call = call;
   rc->data.registered.registered_method = rm;
   rc->data.registered.deadline = deadline;
   rc->initial_metadata = initial_metadata;
   rc->data.registered.optional_payload = optional_payload;
-  error = queue_call_request(&exec_ctx, server, rc);
+  error = queue_call_request(&exec_ctx, server, cq_idx, rc);
 done:
   grpc_exec_ctx_finish(&exec_ctx);
   return error;
 }
 
 static void fail_call(grpc_exec_ctx *exec_ctx, grpc_server *server,
-                      requested_call *rc) {
+                      size_t cq_idx, requested_call *rc) {
   *rc->call = NULL;
   rc->initial_metadata->count = 0;
 
   server_ref(server);
-  grpc_cq_end_op(exec_ctx, rc->cq_for_notification, rc->tag, 0,
-                 done_request_event, rc, &rc->completion);
+  grpc_cq_end_op(exec_ctx, server->cqs[cq_idx], rc->tag, 0, done_request_event,
+                 rc, &rc->completion);
 }
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server) {
diff --git a/src/core/lib/surface/server.h b/src/core/lib/surface/server.h
index 470ef23c690c0921ace6e970dbd88c0ece9b1660..fb6e4d60c501666411b383a9851d6b141c14551c 100644
--- a/src/core/lib/surface/server.h
+++ b/src/core/lib/surface/server.h
@@ -53,6 +53,7 @@ void grpc_server_add_listener(
    server */
 void grpc_server_setup_transport(grpc_exec_ctx *exec_ctx, grpc_server *server,
                                  grpc_transport *transport,
+                                 grpc_pollset *accepting_pollset,
                                  const grpc_channel_args *args);
 
 const grpc_channel_args *grpc_server_get_channel_args(grpc_server *server);
diff --git a/src/core/lib/transport/metadata.c b/src/core/lib/transport/metadata.c
index 5847ec9053d6a24fbf801a1f420703f6b7de6e56..82c8e239f6870c53bbee2dc5792c2d4cd5b52d40 100644
--- a/src/core/lib/transport/metadata.c
+++ b/src/core/lib/transport/metadata.c
@@ -373,7 +373,8 @@ grpc_mdstr *grpc_mdstr_from_buffer(const uint8_t *buf, size_t length) {
     ss = g_static_strtab[idx];
     if (ss == NULL) break;
     if (ss->hash == hash && GPR_SLICE_LENGTH(ss->slice) == length &&
-        0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length)) {
+        (length == 0 ||
+         0 == memcmp(buf, GPR_SLICE_START_PTR(ss->slice), length))) {
       GPR_TIMER_END("grpc_mdstr_from_buffer", 0);
       return ss;
     }
diff --git a/src/cpp/server/server.cc b/src/cpp/server/server.cc
index f955a314944c6a761d3b893b4e07d8e1fb937883..f6c3e5747c9b51020b2afa8c613780a513157be4 100644
--- a/src/cpp/server/server.cc
+++ b/src/cpp/server/server.cc
@@ -295,7 +295,12 @@ Server::Server(ThreadPoolInterface* thread_pool, bool thread_pool_owned,
   grpc_channel_args channel_args;
   args->SetChannelArgs(&channel_args);
   server_ = grpc_server_create(&channel_args, nullptr);
-  grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  if (thread_pool_ == nullptr) {
+    grpc_server_register_non_listening_completion_queue(server_, cq_.cq(),
+                                                        nullptr);
+  } else {
+    grpc_server_register_completion_queue(server_, cq_.cq(), nullptr);
+  }
 }
 
 Server::~Server() {
@@ -407,7 +412,9 @@ bool Server::Start(ServerCompletionQueue** cqs, size_t num_cqs) {
       sync_methods_->push_back(SyncRequest(unknown_method_.get(), nullptr));
     }
     for (size_t i = 0; i < num_cqs; i++) {
-      new UnimplementedAsyncRequest(this, cqs[i]);
+      if (cqs[i]->IsFrequentlyPolled()) {
+        new UnimplementedAsyncRequest(this, cqs[i]);
+      }
     }
   }
   // Start processing rpcs.
diff --git a/src/cpp/server/server_builder.cc b/src/cpp/server/server_builder.cc
index 61f0f6ae2ace14260ba3112e6d2b55f84b0193f0..54feac398253b2569de3f17a51d3ae36acdf1c27 100644
--- a/src/cpp/server/server_builder.cc
+++ b/src/cpp/server/server_builder.cc
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2015-2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -60,8 +60,9 @@ ServerBuilder::ServerBuilder()
   }
 }
 
-std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue() {
-  ServerCompletionQueue* cq = new ServerCompletionQueue();
+std::unique_ptr<ServerCompletionQueue> ServerBuilder::AddCompletionQueue(
+    bool is_frequently_polled) {
+  ServerCompletionQueue* cq = new ServerCompletionQueue(is_frequently_polled);
   cqs_.push_back(cq);
   return std::unique_ptr<ServerCompletionQueue>(cq);
 }
@@ -99,10 +100,12 @@ void ServerBuilder::AddListeningPort(const grpc::string& addr,
 
 std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<ThreadPoolInterface> thread_pool;
+  bool has_sync_methods = false;
   for (auto it = services_.begin(); it != services_.end(); ++it) {
     if ((*it)->service->has_synchronous_methods()) {
       if (thread_pool == nullptr) {
         thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
         break;
       }
     }
@@ -116,6 +119,7 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
     for (auto plugin = plugins_.begin(); plugin != plugins_.end(); plugin++) {
       if ((*plugin).second->has_sync_methods()) {
         thread_pool.reset(CreateDefaultThreadPool());
+        has_sync_methods = true;
         break;
       }
     }
@@ -128,10 +132,33 @@ std::unique_ptr<Server> ServerBuilder::BuildAndStart() {
   std::unique_ptr<Server> server(
       new Server(thread_pool.release(), true, max_message_size_, &args));
   ServerInitializer* initializer = server->initializer();
+
+  // If the server has atleast one sync methods, we know that this is a Sync
+  // server or a Hybrid server and the completion queue (server->cq_) would be
+  // frequently polled.
+  int num_frequently_polled_cqs = has_sync_methods ? 1 : 0;
+
   for (auto cq = cqs_.begin(); cq != cqs_.end(); ++cq) {
-    grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
-                                          nullptr);
+    // A completion queue that is not polled frequently (by calling Next() or
+    // AsyncNext()) is not safe to use for listening to incoming channels.
+    // Register all such completion queues as non-listening completion queues
+    // with the GRPC core library.
+    if ((*cq)->IsFrequentlyPolled()) {
+      grpc_server_register_completion_queue(server->server_, (*cq)->cq(),
+                                            nullptr);
+      num_frequently_polled_cqs++;
+    } else {
+      grpc_server_register_non_listening_completion_queue(server->server_,
+                                                          (*cq)->cq(), nullptr);
+    }
+  }
+
+  if (num_frequently_polled_cqs == 0) {
+    gpr_log(GPR_ERROR,
+            "At least one of the completion queues must be frequently polled");
+    return nullptr;
   }
+
   for (auto service = services_.begin(); service != services_.end();
        service++) {
     if (!server->RegisterService((*service)->host.get(), (*service)->service)) {
diff --git a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
index d92addbf5484dc2bb0f3868ebd20b6f1569114e2..dcdddc769e835059e969a32b2fe831768ec2ee96 100644
--- a/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ClientServerTest.cs
@@ -235,8 +235,16 @@ namespace Grpc.Core.Tests
             await barrier.Task;  // make sure the handler has started.
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
         }
 
         [Test]
@@ -265,9 +273,15 @@ namespace Grpc.Core.Tests
             await handlerStartedBarrier.Task;
             cts.Cancel();
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseAsync);
-            Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
-
+            try
+            {
+                await call.ResponseAsync;
+                Assert.Fail();
+            }
+            catch (RpcException ex)
+            {
+                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+            }
             Assert.AreEqual("SUCCESS", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
index cec8c7ce6bb237468e8a13c2260bfdca353ca068..6a156293ad8d78c4615622a07b3d34e4f4931484 100644
--- a/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
+++ b/src/csharp/Grpc.Core.Tests/ContextPropagationTest.cs
@@ -105,7 +105,15 @@ namespace Grpc.Core.Tests
             var parentCall = Calls.AsyncClientStreamingCall(helper.CreateClientStreamingCall(new CallOptions(cancellationToken: cts.Token)));
             await readyToCancelTcs.Task;
             cts.Cancel();
-            Assert.ThrowsAsync(typeof(RpcException), async () => await parentCall);
+            try
+            {
+                // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                await parentCall;
+                Assert.Fail();
+            }
+            catch (RpcException)
+            {
+            }
             Assert.AreEqual("CHILD_CALL_CANCELLED", await successTcs.Task);
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
index ab12c120cbaf5988eb574f49ce500767779a441e..6fe382751a1d0c38a2dbba2d3c8bf5fe709c5296 100644
--- a/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
+++ b/src/csharp/Grpc.Core.Tests/GrpcEnvironmentTest.cs
@@ -32,7 +32,7 @@
 #endregion
 
 using System;
-using System.Threading;
+using System.Linq;
 using Grpc.Core;
 using NUnit.Framework;
 
@@ -44,7 +44,11 @@ namespace Grpc.Core.Tests
         public void InitializeAndShutdownGrpcEnvironment()
         {
             var env = GrpcEnvironment.AddRef();
-            Assert.IsNotNull(env.CompletionQueue);
+            Assert.IsTrue(env.CompletionQueues.Count > 0);
+            for (int i = 0; i < env.CompletionQueues.Count; i++)
+            {
+                Assert.IsNotNull(env.CompletionQueues.ElementAt(i));
+            }
             GrpcEnvironment.Release();
         }
 
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
index 0e204761f61c0513e25a0a4c0a99090cc3266d3d..c35aaf680f707855454e946ce0007bac749b3f0c 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallServerTest.cs
@@ -53,8 +53,6 @@ namespace Grpc.Core.Internal.Tests
         [SetUp]
         public void Init()
         {
-            var environment = GrpcEnvironment.AddRef();
-
             // Create a fake server just so we have an instance to refer to.
             // The server won't actually be used at all.
             server = new Server()
@@ -66,7 +64,6 @@ namespace Grpc.Core.Internal.Tests
             fakeCall = new FakeNativeCall();
             asyncCallServer = new AsyncCallServer<string, string>(
                 Marshallers.StringMarshaller.Serializer, Marshallers.StringMarshaller.Deserializer,
-                environment,
                 server);
             asyncCallServer.InitializeForTesting(fakeCall);
         }
@@ -75,7 +72,6 @@ namespace Grpc.Core.Internal.Tests
         public void Cleanup()
         {
             server.ShutdownAsync().Wait();
-            GrpcEnvironment.Release();
         }
 
         [Test]
@@ -136,7 +132,6 @@ namespace Grpc.Core.Internal.Tests
         public void WriteAfterCancelNotificationFails()
         {
             var finishedTask = asyncCallServer.ServerSideCallAsync();
-            var requestStream = new ServerRequestStream<string, string>(asyncCallServer);
             var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
 
             fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
@@ -181,6 +176,21 @@ namespace Grpc.Core.Internal.Tests
             AssertFinished(asyncCallServer, fakeCall, finishedTask);
         }
 
+        [Test]
+        public void WriteAfterWriteStatusThrowsInvalidOperationException()
+        {
+            var finishedTask = asyncCallServer.ServerSideCallAsync();
+            var responseStream = new ServerResponseStream<string, string>(asyncCallServer);
+
+            asyncCallServer.SendStatusFromServerAsync(Status.DefaultSuccess, new Metadata(), null);
+            Assert.ThrowsAsync(typeof(InvalidOperationException), async () => await responseStream.WriteAsync("request1"));
+
+            fakeCall.SendStatusFromServerHandler(true);
+            fakeCall.ReceivedCloseOnServerHandler(true, cancelled: true);
+
+            AssertFinished(asyncCallServer, fakeCall, finishedTask);
+        }
+
         static void AssertFinished(AsyncCallServer<string, string> asyncCallServer, FakeNativeCall fakeCall, Task finishedTask)
         {
             Assert.IsTrue(fakeCall.IsDisposed);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
index 777a1c8c500d9487b221b24142cd6b21a87ec52c..81897f8c77236d7e64149b2669fba05021a83ea7 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/AsyncCallTest.cs
@@ -33,7 +33,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
 using System.Threading.Tasks;
 
 using Grpc.Core.Internal;
@@ -82,7 +81,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.ThrowsAsync(typeof(InvalidOperationException),
                 async () => await asyncCall.ReadMessageAsync());
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -103,7 +102,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.UnaryCallAsync("request1");
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -148,7 +147,7 @@ namespace Grpc.Core.Internal.Tests
             var resultTask = asyncCall.ClientStreamingCallAsync();
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.InvalidArgument),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.InvalidArgument);
@@ -193,7 +192,7 @@ namespace Grpc.Core.Internal.Tests
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Internal),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Internal);
@@ -211,7 +210,9 @@ namespace Grpc.Core.Internal.Tests
                 new Metadata());
 
             AssertUnaryResponseSuccess(asyncCall, fakeCall, resultTask);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
@@ -223,11 +224,13 @@ namespace Grpc.Core.Internal.Tests
 
             fakeCall.UnaryResponseClientHandler(true,
                 new ClientSideStatus(new Status(StatusCode.OutOfRange, ""), new Metadata()),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.OutOfRange);
-            var ex = Assert.Throws<RpcException>(() => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(StatusCode.OutOfRange, ex.Status.StatusCode);
         }
 
@@ -267,7 +270,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void ClientStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void ClientStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             var resultTask = asyncCall.ClientStreamingCallAsync();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -275,11 +278,12 @@ namespace Grpc.Core.Internal.Tests
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
 
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             fakeCall.UnaryResponseClientHandler(true,
                 CreateClientSideStatus(StatusCode.Cancelled),
-                CreateResponsePayload(),
+                null,
                 new Metadata());
 
             AssertUnaryResponseError(asyncCall, fakeCall, resultTask, StatusCode.Cancelled);
@@ -290,7 +294,7 @@ namespace Grpc.Core.Internal.Tests
         {
             asyncCall.StartServerStreamingCall("request1");
             Assert.Throws(typeof(InvalidOperationException),
-                () => asyncCall.StartSendMessage("abc", new WriteFlags(), (x,y) => {}));
+                () => asyncCall.SendMessageAsync("abc", new WriteFlags()));
         }
 
         [Test]
@@ -390,12 +394,13 @@ namespace Grpc.Core.Internal.Tests
 
             AssertStreamingResponseSuccess(asyncCall, fakeCall, readTask);
 
-            var ex = Assert.ThrowsAsync<RpcException>(async () => await requestStream.WriteAsync("request1"));
+            var writeTask = requestStream.WriteAsync("request1");
+            var ex = Assert.ThrowsAsync<RpcException>(async () => await writeTask);
             Assert.AreEqual(Status.DefaultSuccess, ex.Status);
         }
 
         [Test]
-        public void DuplexStreaming_CompleteAfterReceivingStatusFails()
+        public void DuplexStreaming_CompleteAfterReceivingStatusSuceeds()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -411,7 +416,7 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void DuplexStreaming_WriteAfterCancellationRequestThrowsOperationCancelledException()
+        public void DuplexStreaming_WriteAfterCancellationRequestThrowsTaskCanceledException()
         {
             asyncCall.StartDuplexStreamingCall();
             var requestStream = new ClientRequestStream<string, string>(asyncCall);
@@ -419,7 +424,9 @@ namespace Grpc.Core.Internal.Tests
 
             asyncCall.Cancel();
             Assert.IsTrue(fakeCall.IsCancelled);
-            Assert.Throws(typeof(OperationCanceledException), () => requestStream.WriteAsync("request1"));
+
+            var writeTask = requestStream.WriteAsync("request1");
+            Assert.ThrowsAsync(typeof(TaskCanceledException), async () => await writeTask);
 
             var readTask = responseStream.MoveNext();
             fakeCall.ReceivedMessageHandler(true, null);
diff --git a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
index c6843f10af25744153b8da6095eade1c4d260d92..195119f920d99da41549fa90b2a13c552c0e072d 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/CompletionQueueSafeHandleTest.cs
@@ -60,7 +60,7 @@ namespace Grpc.Core.Internal.Tests
             var ev = cq.Next();
             cq.Dispose();
             GrpcEnvironment.Release();
-            Assert.AreEqual(GRPCCompletionType.Shutdown, ev.type);
+            Assert.AreEqual(CompletionQueueEvent.CompletionType.Shutdown, ev.type);
             Assert.AreNotEqual(IntPtr.Zero, ev.success);
             Assert.AreEqual(IntPtr.Zero, ev.tag);
         }
diff --git a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
index 74f7f2497a92ddee88556dd81f81a8932a221ba3..c124ea29af8e875bd0908953876c8fbc0cace230 100644
--- a/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
+++ b/src/csharp/Grpc.Core.Tests/Internal/TimespecTest.cs
@@ -61,15 +61,15 @@ namespace Grpc.Core.Internal.Tests
         }
 
         [Test]
-        public void InfFuture()
+        public void InfFutureMatchesNativeValue()
         {
-            var timespec = Timespec.InfFuture;
+            Assert.AreEqual(Timespec.NativeInfFuture, Timespec.InfFuture);
         }
 
         [Test]
-        public void InfPast()
+        public void InfPastMatchesNativeValue()
         {
-            var timespec = Timespec.InfPast;
+            Assert.AreEqual(Timespec.NativeInfPast, Timespec.InfPast);
         }
 
         [Test]
@@ -108,7 +108,7 @@ namespace Grpc.Core.Internal.Tests
             Assert.Throws(typeof(InvalidOperationException),
                 () => new Timespec(0, 1000 * 1000 * 1000).ToDateTime());
             Assert.Throws(typeof(InvalidOperationException),
-                () => new Timespec(0, 0, GPRClockType.Monotonic).ToDateTime());
+                () => new Timespec(0, 0, ClockType.Monotonic).ToDateTime());
         }
 
         [Test]
diff --git a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
index 0663e77d1eaf7c8e6db9708b036d61ed172695ed..d770f82390d4b54ddeac4cfd6b88836a5151e0d2 100644
--- a/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
+++ b/src/csharp/Grpc.Core.Tests/MarshallingErrorsTest.cs
@@ -134,7 +134,15 @@ namespace Grpc.Core.Tests
         {
             helper.ClientStreamingHandler = new ClientStreamingServerMethod<string, string>(async (requestStream, context) =>
             {
-                Assert.ThrowsAsync<IOException>(async () => await requestStream.MoveNext());
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await requestStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (IOException)
+                {
+                }
                 return "RESPONSE";
             });
 
diff --git a/src/csharp/Grpc.Core/Channel.cs b/src/csharp/Grpc.Core/Channel.cs
index 93a6e6a3d95fee3456c07b0c1c42f4e9ab29ee27..886adfec334f8c8711ebedb98346190337a7a6be 100644
--- a/src/csharp/Grpc.Core/Channel.cs
+++ b/src/csharp/Grpc.Core/Channel.cs
@@ -31,7 +31,6 @@
 
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 
@@ -56,6 +55,7 @@ namespace Grpc.Core
 
         readonly string target;
         readonly GrpcEnvironment environment;
+        readonly CompletionQueueSafeHandle completionQueue;
         readonly ChannelSafeHandle handle;
         readonly Dictionary<string, ChannelOption> options;
 
@@ -75,6 +75,7 @@ namespace Grpc.Core
             EnsureUserAgentChannelOption(this.options);
             this.environment = GrpcEnvironment.AddRef();
 
+            this.completionQueue = this.environment.PickCompletionQueue();
             using (var nativeCredentials = credentials.ToNativeCredentials())
             using (var nativeChannelArgs = ChannelOptions.CreateChannelArgs(this.options.Values))
             {
@@ -135,7 +136,7 @@ namespace Grpc.Core
                     tcs.SetCanceled();
                 }
             });
-            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, environment.CompletionQueue, environment.CompletionRegistry, handler);
+            handle.WatchConnectivityState(lastObservedState, deadlineTimespec, completionQueue, handler);
             return tcs.Task;
         }
 
@@ -231,6 +232,14 @@ namespace Grpc.Core
             }
         }
 
+        internal CompletionQueueSafeHandle CompletionQueue
+        {
+            get
+            {
+                return this.completionQueue;
+            }
+        }
+
         internal void AddCallReference(object call)
         {
             activeCallCounter.Increment();
diff --git a/src/csharp/Grpc.Core/Grpc.Core.csproj b/src/csharp/Grpc.Core/Grpc.Core.csproj
index 95077a6ca5a2c2f87a93eceae8506017bd430841..a8b7b5f00d9ed846c3635bcd4e52386892854660 100644
--- a/src/csharp/Grpc.Core/Grpc.Core.csproj
+++ b/src/csharp/Grpc.Core/Grpc.Core.csproj
@@ -74,7 +74,6 @@
     <Compile Include="Internal\CallSafeHandle.cs" />
     <Compile Include="Internal\ChannelSafeHandle.cs" />
     <Compile Include="Internal\CompletionQueueSafeHandle.cs" />
-    <Compile Include="Internal\Enums.cs" />
     <Compile Include="Internal\SafeHandleZeroIsInvalid.cs" />
     <Compile Include="Internal\Timespec.cs" />
     <Compile Include="Internal\GrpcThreadPool.cs" />
@@ -87,7 +86,6 @@
     <Compile Include="Utils\BenchmarkUtil.cs" />
     <Compile Include="ChannelCredentials.cs" />
     <Compile Include="Internal\ChannelArgsSafeHandle.cs" />
-    <Compile Include="Internal\AsyncCompletion.cs" />
     <Compile Include="Internal\AsyncCallBase.cs" />
     <Compile Include="Internal\AsyncCallServer.cs" />
     <Compile Include="Internal\AsyncCall.cs" />
@@ -134,6 +132,10 @@
     <Compile Include="DefaultCallInvoker.cs" />
     <Compile Include="Internal\UnimplementedCallInvoker.cs" />
     <Compile Include="Internal\InterceptingCallInvoker.cs" />
+    <Compile Include="Internal\ServerRpcNew.cs" />
+    <Compile Include="Internal\ClientSideStatus.cs" />
+    <Compile Include="Internal\ClockType.cs" />
+    <Compile Include="Internal\CallError.cs" />
   </ItemGroup>
   <ItemGroup>
     <None Include="Grpc.Core.nuspec" />
diff --git a/src/csharp/Grpc.Core/GrpcEnvironment.cs b/src/csharp/Grpc.Core/GrpcEnvironment.cs
index bee0ef1d62e08f1f7222fc0e704fce363e151c76..18af1099f1b33999550ef28de65110470bc5cd5b 100644
--- a/src/csharp/Grpc.Core/GrpcEnvironment.cs
+++ b/src/csharp/Grpc.Core/GrpcEnvironment.cs
@@ -32,8 +32,9 @@
 #endregion
 
 using System;
+using System.Collections.Generic;
+using System.Linq;
 using System.Runtime.InteropServices;
-using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
 using Grpc.Core.Utils;
@@ -51,12 +52,13 @@ namespace Grpc.Core
         static GrpcEnvironment instance;
         static int refCount;
         static int? customThreadPoolSize;
+        static int? customCompletionQueueCount;
 
         static ILogger logger = new ConsoleLogger();
 
         readonly GrpcThreadPool threadPool;
-        readonly CompletionRegistry completionRegistry;
         readonly DebugStats debugStats = new DebugStats();
+        readonly AtomicCounter cqPickerCounter = new AtomicCounter();
         bool isClosed;
 
         /// <summary>
@@ -140,37 +142,52 @@ namespace Grpc.Core
             }
         }
 
+        /// <summary>
+        /// Sets the number of completion queues in the  gRPC thread pool that polls for internal RPC events.
+        /// Can be only invoke before the <c>GrpcEnviroment</c> is started and cannot be changed afterwards.
+        /// Setting the number of completions queues is an advanced setting and you should only use it if you know what you are doing.
+        /// Most users should rely on the default value provided by gRPC library.
+        /// Note: this method is part of an experimental API that can change or be removed without any prior notice.
+        /// </summary>
+        public static void SetCompletionQueueCount(int completionQueueCount)
+        {
+            lock (staticLock)
+            {
+                GrpcPreconditions.CheckState(instance == null, "Can only be set before GrpcEnvironment is initialized");
+                GrpcPreconditions.CheckArgument(completionQueueCount > 0, "threadCount needs to be a positive number");
+                customCompletionQueueCount = completionQueueCount;
+            }
+        }
+
         /// <summary>
         /// Creates gRPC environment.
         /// </summary>
         private GrpcEnvironment()
         {
             GrpcNativeInit();
-            completionRegistry = new CompletionRegistry(this);
-            threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault());
+            threadPool = new GrpcThreadPool(this, GetThreadPoolSizeOrDefault(), GetCompletionQueueCountOrDefault());
             threadPool.Start();
         }
 
         /// <summary>
-        /// Gets the completion registry used by this gRPC environment.
+        /// Gets the completion queues used by this gRPC environment.
         /// </summary>
-        internal CompletionRegistry CompletionRegistry
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return this.completionRegistry;
+                return this.threadPool.CompletionQueues;
             }
         }
 
         /// <summary>
-        /// Gets the completion queue used by this gRPC environment.
+        /// Picks a completion queue in a round-robin fashion.
+        /// Shouldn't be invoked on a per-call basis (used at per-channel basis).
         /// </summary>
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal CompletionQueueSafeHandle PickCompletionQueue()
         {
-            get
-            {
-                return this.threadPool.CompletionQueue;
-            }
+            var cqIndex = (int) ((cqPickerCounter.Increment() - 1) % this.threadPool.CompletionQueues.Count);
+            return this.threadPool.CompletionQueues.ElementAt(cqIndex);
         }
 
         /// <summary>
@@ -230,5 +247,15 @@ namespace Grpc.Core
             // more work, but seems to work reasonably well for a start.
             return Math.Max(MinDefaultThreadPoolSize, Environment.ProcessorCount / 2);
         }
+
+        private int GetCompletionQueueCountOrDefault()
+        {
+            if (customCompletionQueueCount.HasValue)
+            {
+                return customCompletionQueueCount.Value;
+            }
+            // by default, create a completion queue for each thread
+            return GetThreadPoolSizeOrDefault();
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCall.cs b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
index 55351869b5c9948b76b65d71ee215efc9d2c8640..895be690a590445087ef96707104cb0da7f97042 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCall.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCall.cs
@@ -32,12 +32,7 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
 using System.Threading.Tasks;
-using Grpc.Core.Internal;
 using Grpc.Core.Logging;
 using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
@@ -57,9 +52,11 @@ namespace Grpc.Core.Internal
         // Completion of a pending unary response if not null.
         TaskCompletionSource<TResponse> unaryResponseTcs;
 
+        // TODO(jtattermusch): this field doesn't need to be initialized for unary response calls.
         // Indicates that response streaming call has finished.
         TaskCompletionSource<object> streamingCallFinishedTcs = new TaskCompletionSource<object>();
 
+        // TODO(jtattermusch): this field could be lazy-initialized (only if someone requests the response headers).
         // Response headers set here once received.
         TaskCompletionSource<Metadata> responseHeadersTcs = new TaskCompletionSource<Metadata>();
 
@@ -67,7 +64,7 @@ namespace Grpc.Core.Internal
         ClientSideStatus? finishedStatus;
 
         public AsyncCall(CallInvocationDetails<TRequest, TResponse> callDetails)
-            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer, callDetails.Channel.Environment)
+            : base(callDetails.RequestMarshaller.Serializer, callDetails.ResponseMarshaller.Deserializer)
         {
             this.details = callDetails.WithOptions(callDetails.Options.Normalize());
             this.initialMetadataSent = true;  // we always send metadata at the very beginning of the call.
@@ -144,7 +141,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
                 readingDone = true;
@@ -171,7 +168,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 readingDone = true;
 
@@ -195,7 +192,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 halfcloseRequested = true;
 
@@ -220,7 +217,7 @@ namespace Grpc.Core.Internal
                 GrpcPreconditions.CheckState(!started);
                 started = true;
 
-                Initialize(environment.CompletionQueue);
+                Initialize(details.Channel.CompletionQueue);
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(details.Options.Headers))
                 {
@@ -232,11 +229,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming request. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TRequest msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TRequest msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -250,29 +246,32 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Sends halfclose, indicating client is done with streaming requests.
         /// Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendCloseFromClient(AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendCloseFromClientAsync()
         {
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: true);
+                GrpcPreconditions.CheckState(started);
 
-                if (!disposed && !finished)
+                var earlyResult = CheckSendPreconditionsClientSide();
+                if (earlyResult != null)
                 {
-                    call.StartSendCloseFromClient(HandleSendCloseFromClientFinished);
+                    return earlyResult;
                 }
-                else
+
+                if (disposed || finished)
                 {
                     // In case the call has already been finished by the serverside,
-                    // the halfclose has already been done implicitly, so we only
-                    // emit the notification for the completion delegate.
-                    Task.Run(() => HandleSendCloseFromClientFinished(true));
+                    // the halfclose has already been done implicitly, so just return
+                    // completed task here.
+                    halfcloseRequested = true;
+                    return Task.FromResult<object>(null);
                 }
+                call.StartSendCloseFromClient(HandleSendCloseFromClientFinished);
 
                 halfcloseRequested = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -342,6 +341,45 @@ namespace Grpc.Core.Internal
             get { return true; }
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            var earlyResult = CheckSendPreconditionsClientSide();
+            if (earlyResult != null)
+            {
+                return earlyResult;
+            }
+
+            if (finishedStatus.HasValue)
+            {
+                // throwing RpcException if we already received status on client
+                // side makes the most sense.
+                // Note that this throws even for StatusCode.OK.
+                // Writing after the call has finished is not a programming error because server can close
+                // the call anytime, so don't throw directly, but let the write task finish with an error.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetException(new RpcException(finishedStatus.Value.Status));
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
+        private Task CheckSendPreconditionsClientSide()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Request stream has already been completed.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time.");
+
+            if (cancelRequested)
+            {
+                // Return a cancelled task.
+                var tcs = new TaskCompletionSource<object>();
+                tcs.SetCanceled();
+                return tcs.Task;
+            }
+
+            return null;
+        }
+
         private void Initialize(CompletionQueueSafeHandle cq)
         {
             using (Profilers.ForCurrentThread().NewScope("AsyncCall.Initialize"))
@@ -368,7 +406,7 @@ namespace Grpc.Core.Internal
                 var credentials = details.Options.Credentials;
                 using (var nativeCredentials = credentials != null ? credentials.ToNativeCredentials() : null)
                 {
-                    var result = details.Channel.Handle.CreateCall(environment.CompletionRegistry,
+                    var result = details.Channel.Handle.CreateCall(
                                  parentCall, ContextPropagationToken.DefaultMask, cq,
                                  details.Method, details.Host, Timespec.FromDateTime(details.Options.Deadline.Value), nativeCredentials);
                     return result;
@@ -400,6 +438,7 @@ namespace Grpc.Core.Internal
         /// </summary>
         private void HandleReceivedResponseHeaders(bool success, Metadata responseHeaders)
         {
+            // TODO(jtattermusch): handle success==false
             responseHeadersTcs.SetResult(responseHeaders);
         }
 
@@ -443,19 +482,6 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected override void CheckSendingAllowed(bool allowFinished)
-        {
-            base.CheckSendingAllowed(true);
-
-            // throwing RpcException if we already received status on client
-            // side makes the most sense.
-            // Note that this throws even for StatusCode.OK.
-            if (!allowFinished && finishedStatus.HasValue)
-            {
-                throw new RpcException(finishedStatus.Value.Status);
-            }
-        }
-
         /// <summary>
         /// Handles receive status completion for calls with streaming response.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
index 4de23706b28e5989e6288eb48d3fff7193be363a..cb8366c21666d236d441fcbf545678010e00a5a5 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallBase.cs
@@ -58,7 +58,6 @@ namespace Grpc.Core.Internal
         readonly Func<TWrite, byte[]> serializer;
         readonly Func<byte[], TRead> deserializer;
 
-        protected readonly GrpcEnvironment environment;
         protected readonly object myLock = new object();
 
         protected INativeCall call;
@@ -67,8 +66,8 @@ namespace Grpc.Core.Internal
         protected bool started;
         protected bool cancelRequested;
 
-        protected AsyncCompletionDelegate<object> sendCompletionDelegate;  // Completion of a pending send or sendclose if not null.
         protected TaskCompletionSource<TRead> streamingReadTcs;  // Completion of a pending streaming read if not null.
+        protected TaskCompletionSource<object> streamingWriteTcs;  // Completion of a pending streaming write or send close from client if not null.
         protected TaskCompletionSource<object> sendStatusFromServerTcs;
 
         protected bool readingDone;  // True if last read (i.e. read with null payload) was already received.
@@ -78,11 +77,10 @@ namespace Grpc.Core.Internal
         protected bool initialMetadataSent;
         protected long streamingWritesCounter;  // Number of streaming send operations started so far.
 
-        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer, GrpcEnvironment environment)
+        public AsyncCallBase(Func<TWrite, byte[]> serializer, Func<byte[], TRead> deserializer)
         {
             this.serializer = GrpcPreconditions.CheckNotNull(serializer);
             this.deserializer = GrpcPreconditions.CheckNotNull(deserializer);
-            this.environment = GrpcPreconditions.CheckNotNull(environment);
         }
 
         /// <summary>
@@ -128,28 +126,31 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Initiates sending a message. Only one send operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        protected void StartSendMessageInternal(TWrite msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        protected Task SendMessageInternalAsync(TWrite msg, WriteFlags writeFlags)
         {
             byte[] payload = UnsafeSerialize(msg);
 
             lock (myLock)
             {
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
-                CheckSendingAllowed(allowFinished: false);
+                GrpcPreconditions.CheckState(started);
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 call.StartSendMessage(HandleSendFinished, payload, writeFlags, !initialMetadataSent);
 
-                sendCompletionDelegate = completionDelegate;
                 initialMetadataSent = true;
                 streamingWritesCounter++;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
         /// <summary>
         /// Initiates reading a message. Only one read operation can be active at a time.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
         protected Task<TRead> ReadMessageInternalAsync()
         {
@@ -159,7 +160,7 @@ namespace Grpc.Core.Internal
                 if (readingDone)
                 {
                     // the last read that returns null or throws an exception is idempotent
-                    // and maintain its state.
+                    // and maintains its state.
                     GrpcPreconditions.CheckState(streamingReadTcs != null, "Call does not support streaming reads.");
                     return streamingReadTcs.Task;
                 }
@@ -183,7 +184,7 @@ namespace Grpc.Core.Internal
             {
                 if (!disposed && call != null)
                 {
-                    bool noMoreSendCompletions = sendCompletionDelegate == null && (halfcloseRequested || cancelRequested || finished);
+                    bool noMoreSendCompletions = streamingWriteTcs == null && (halfcloseRequested || cancelRequested || finished);
                     if (noMoreSendCompletions && readingDone && finished)
                     {
                         ReleaseResources();
@@ -213,24 +214,11 @@ namespace Grpc.Core.Internal
         {
         }
 
-        protected virtual void CheckSendingAllowed(bool allowFinished)
-        {
-            GrpcPreconditions.CheckState(started);
-            CheckNotCancelled();
-            GrpcPreconditions.CheckState(!disposed || allowFinished);
-
-            GrpcPreconditions.CheckState(!halfcloseRequested, "Already halfclosed.");
-            GrpcPreconditions.CheckState(!finished || allowFinished, "Already finished.");
-            GrpcPreconditions.CheckState(sendCompletionDelegate == null, "Only one write can be pending at a time");
-        }
-
-        protected void CheckNotCancelled()
-        {
-            if (cancelRequested)
-            {
-                throw new OperationCanceledException("Remote call has been cancelled.");
-            }
-        }
+        /// <summary>
+        /// Checks if sending is allowed and possibly returns a Task that allows short-circuiting the send
+        /// logic by directly returning the write operation result task. Normally, null is returned.
+        /// </summary>
+        protected abstract Task CheckSendAllowedOrEarlyResult();
 
         protected byte[] UnsafeSerialize(TWrite msg)
         {
@@ -259,39 +247,27 @@ namespace Grpc.Core.Internal
             }
         }
 
-        protected void FireCompletion<T>(AsyncCompletionDelegate<T> completionDelegate, T value, Exception error)
-        {
-            try
-            {
-                completionDelegate(value, error);
-            }
-            catch (Exception e)
-            {
-                Logger.Error(e, "Exception occured while invoking completion delegate.");
-            }
-        }
-
         /// <summary>
         /// Handles send completion.
         /// </summary>
         protected void HandleSendFinished(bool success)
         {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
+            TaskCompletionSource<object> origTcs = null;
             lock (myLock)
             {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
+                origTcs = streamingWriteTcs;
+                streamingWriteTcs = null;
 
                 ReleaseResourcesIfPossible();
             }
 
             if (!success)
             {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Send failed"));
+                origTcs.SetException(new InvalidOperationException("Send failed"));
             }
             else
             {
-                FireCompletion(origCompletionDelegate, null, null);
+                origTcs.SetResult(null);
             }
         }
 
@@ -300,22 +276,23 @@ namespace Grpc.Core.Internal
         /// </summary>
         protected void HandleSendCloseFromClientFinished(bool success)
         {
-            AsyncCompletionDelegate<object> origCompletionDelegate = null;
+            TaskCompletionSource<object> origTcs = null;
             lock (myLock)
             {
-                origCompletionDelegate = sendCompletionDelegate;
-                sendCompletionDelegate = null;
+                origTcs = streamingWriteTcs;
+                streamingWriteTcs = null;
 
                 ReleaseResourcesIfPossible();
             }
 
             if (!success)
             {
-                FireCompletion(origCompletionDelegate, null, new InvalidOperationException("Sending close from client has failed."));
+                // TODO(jtattermusch): this method is same as HandleSendFinished (only the error message differs).
+                origTcs.SetException(new InvalidOperationException("Sending close from client has failed."));
             }
             else
             {
-                FireCompletion(origCompletionDelegate, null, null);
+                origTcs.SetResult(null);
             }
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
index b1566b44a7ccd212b015b65532493d0280743998..56c23ba3ef3b00f3377cc122e6508cea6b38e400 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
+++ b/src/csharp/Grpc.Core/Internal/AsyncCallServer.cs
@@ -51,14 +51,14 @@ namespace Grpc.Core.Internal
         readonly CancellationTokenSource cancellationTokenSource = new CancellationTokenSource();
         readonly Server server;
 
-        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, GrpcEnvironment environment, Server server) : base(serializer, deserializer, environment)
+        public AsyncCallServer(Func<TResponse, byte[]> serializer, Func<byte[], TRequest> deserializer, Server server) : base(serializer, deserializer)
         {
             this.server = GrpcPreconditions.CheckNotNull(server);
         }
 
-        public void Initialize(CallSafeHandle call)
+        public void Initialize(CallSafeHandle call, CompletionQueueSafeHandle completionQueue)
         {
-            call.Initialize(environment.CompletionRegistry, environment.CompletionQueue);
+            call.Initialize(completionQueue);
 
             server.AddCallReference(this);
             InitializeInternal(call);
@@ -91,11 +91,10 @@ namespace Grpc.Core.Internal
 
         /// <summary>
         /// Sends a streaming response. Only one pending send action is allowed at any given time.
-        /// completionDelegate is called when the operation finishes.
         /// </summary>
-        public void StartSendMessage(TResponse msg, WriteFlags writeFlags, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendMessageAsync(TResponse msg, WriteFlags writeFlags)
         {
-            StartSendMessageInternal(msg, writeFlags, completionDelegate);
+            return SendMessageInternalAsync(msg, writeFlags);
         }
 
         /// <summary>
@@ -110,20 +109,22 @@ namespace Grpc.Core.Internal
         /// Initiates sending a initial metadata. 
         /// Even though C-core allows sending metadata in parallel to sending messages, we will treat sending metadata as a send message operation
         /// to make things simpler.
-        /// completionDelegate is invoked upon completion.
         /// </summary>
-        public void StartSendInitialMetadata(Metadata headers, AsyncCompletionDelegate<object> completionDelegate)
+        public Task SendInitialMetadataAsync(Metadata headers)
         {
             lock (myLock)
             {
                 GrpcPreconditions.CheckNotNull(headers, "metadata");
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
 
+                GrpcPreconditions.CheckState(started);
                 GrpcPreconditions.CheckState(!initialMetadataSent, "Response headers can only be sent once per call.");
                 GrpcPreconditions.CheckState(streamingWritesCounter == 0, "Response headers can only be sent before the first write starts.");
-                CheckSendingAllowed(allowFinished: false);
 
-                GrpcPreconditions.CheckNotNull(completionDelegate, "Completion delegate cannot be null");
+                var earlyResult = CheckSendAllowedOrEarlyResult();
+                if (earlyResult != null)
+                {
+                    return earlyResult;
+                }
 
                 using (var metadataArray = MetadataArraySafeHandle.Create(headers))
                 {
@@ -131,7 +132,8 @@ namespace Grpc.Core.Internal
                 }
 
                 this.initialMetadataSent = true;
-                sendCompletionDelegate = completionDelegate;
+                streamingWriteTcs = new TaskCompletionSource<object>();
+                return streamingWriteTcs.Task;
             }
         }
 
@@ -196,6 +198,16 @@ namespace Grpc.Core.Internal
             server.RemoveCallReference(this);
         }
 
+        protected override Task CheckSendAllowedOrEarlyResult()
+        {
+            GrpcPreconditions.CheckState(!halfcloseRequested, "Response stream has already been completed.");
+            GrpcPreconditions.CheckState(!finished, "Already finished.");
+            GrpcPreconditions.CheckState(streamingWriteTcs == null, "Only one write can be pending at a time");
+            GrpcPreconditions.CheckState(!disposed);
+
+            return null;
+        }
+
         /// <summary>
         /// Handles the server side close completion.
         /// </summary>
diff --git a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
index 66d2a66f99f1af8d9f3ddee2fc0c686f8869504f..c28a6f64d39f3a4b170d06247f51b41466bb9556 100644
--- a/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/BatchContextSafeHandle.cs
@@ -120,107 +120,4 @@ namespace Grpc.Core.Internal
             return true;
         }
     }
-
-    /// <summary>
-    /// Status + metadata received on client side when call finishes.
-    /// (when receive_status_on_client operation finishes).
-    /// </summary>
-    internal struct ClientSideStatus
-    {
-        readonly Status status;
-        readonly Metadata trailers;
-
-        public ClientSideStatus(Status status, Metadata trailers)
-        {
-            this.status = status;
-            this.trailers = trailers;
-        }
-
-        public Status Status
-        {
-            get
-            {
-                return this.status;
-            }    
-        }
-
-        public Metadata Trailers
-        {
-            get
-            {
-                return this.trailers;
-            }
-        }
-    }
-
-    /// <summary>
-    /// Details of a newly received RPC.
-    /// </summary>
-    internal struct ServerRpcNew
-    {
-        readonly Server server;
-        readonly CallSafeHandle call;
-        readonly string method;
-        readonly string host;
-        readonly Timespec deadline;
-        readonly Metadata requestMetadata;
-
-        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
-        {
-            this.server = server;
-            this.call = call;
-            this.method = method;
-            this.host = host;
-            this.deadline = deadline;
-            this.requestMetadata = requestMetadata;
-        }
-
-        public Server Server
-        {
-            get
-            {
-                return this.server;
-            }
-        }
-
-        public CallSafeHandle Call
-        {
-            get
-            {
-                return this.call;
-            }
-        }
-
-        public string Method
-        {
-            get
-            {
-                return this.method;
-            }
-        }
-
-        public string Host
-        {
-            get
-            {
-                return this.host;
-            }
-        }
-
-        public Timespec Deadline
-        {
-            get
-            {
-                return this.deadline;
-            }
-        }
-
-        public Metadata RequestMetadata
-        {
-            get
-            {
-                return this.requestMetadata;
-            }
-        }
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/Enums.cs b/src/csharp/Grpc.Core/Internal/CallError.cs
similarity index 76%
rename from src/csharp/Grpc.Core/Internal/Enums.cs
rename to src/csharp/Grpc.Core/Internal/CallError.cs
index 74f86d2a3044a29178a58bf8376b533d8c82b7b1..541575f5e64c3ec6a5e65aa09c3b20e306d5a72c 100644
--- a/src/csharp/Grpc.Core/Internal/Enums.cs
+++ b/src/csharp/Grpc.Core/Internal/CallError.cs
@@ -40,7 +40,7 @@ namespace Grpc.Core.Internal
     /// <summary>
     /// grpc_call_error from grpc/grpc.h
     /// </summary>
-    internal enum GRPCCallError
+    internal enum CallError
     {
         /* everything went ok */
         OK = 0,
@@ -70,42 +70,9 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Checks the call API invocation's result is OK.
         /// </summary>
-        public static void CheckOk(this GRPCCallError callError)
+        public static void CheckOk(this CallError callError)
         {
-            GrpcPreconditions.CheckState(callError == GRPCCallError.OK, "Call error: " + callError);
+            GrpcPreconditions.CheckState(callError == CallError.OK, "Call error: " + callError);
         }
     }
-
-    /// <summary>
-    /// grpc_completion_type from grpc/grpc.h
-    /// </summary>
-    internal enum GRPCCompletionType
-    {
-        /* Shutting down */
-        Shutdown, 
-
-        /* No event before timeout */
-        Timeout,  
-
-        /* operation completion */
-        OpComplete
-    }
-
-    /// <summary>
-    /// gpr_clock_type from grpc/support/time.h
-    /// </summary>
-    internal enum GPRClockType
-    {
-        /* Monotonic clock */
-        Monotonic,
-
-        /* Realtime clock */
-        Realtime,
-
-        /* Precise clock good for performance profiling. */
-        Precise,
-
-        /* Timespan - the distance between two time points */
-        Timespan
-    }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
index 244b97d4a47de04739e05ce5f7e054394c3b9cf7..82361f57979acc62e2c3c2a82e8f534bf0a7ba3d 100644
--- a/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CallSafeHandle.cs
@@ -47,16 +47,14 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         const uint GRPC_WRITE_BUFFER_HINT = 1;
-        CompletionRegistry completionRegistry;
         CompletionQueueSafeHandle completionQueue;
 
         private CallSafeHandle()
         {
         }
 
-        public void Initialize(CompletionRegistry completionRegistry, CompletionQueueSafeHandle completionQueue)
+        public void Initialize(CompletionQueueSafeHandle completionQueue)
         {
-            this.completionRegistry = completionRegistry;
             this.completionQueue = completionQueue;
         }
 
@@ -70,7 +68,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_unary(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags)
                     .CheckOk();
             }
@@ -90,7 +88,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient(), context.GetReceivedMessage(), context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_start_client_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -100,7 +98,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_server_streaming(this, ctx, payload, new UIntPtr((ulong)payload.Length), metadataArray, writeFlags).CheckOk();
             }
         }
@@ -110,7 +108,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedStatusOnClient()));
                 Native.grpcsharp_call_start_duplex_streaming(this, ctx, metadataArray).CheckOk();
             }
         }
@@ -120,7 +118,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_message(this, ctx, payload, new UIntPtr((ulong)payload.Length), writeFlags, sendEmptyInitialMetadata).CheckOk();
             }
         }
@@ -130,7 +128,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_close_from_client(this, ctx).CheckOk();
             }
         }
@@ -142,7 +140,7 @@ namespace Grpc.Core.Internal
             {
                 var ctx = BatchContextSafeHandle.Create();
                 var optionalPayloadLength = optionalPayload != null ? new UIntPtr((ulong)optionalPayload.Length) : UIntPtr.Zero;
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_status_from_server(this, ctx, status.StatusCode, status.Detail, metadataArray, sendEmptyInitialMetadata,
                     optionalPayload, optionalPayloadLength, writeFlags).CheckOk();
             }
@@ -153,7 +151,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedMessage()));
                 Native.grpcsharp_call_recv_message(this, ctx).CheckOk();
             }
         }
@@ -163,7 +161,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedInitialMetadata()));
                 Native.grpcsharp_call_recv_initial_metadata(this, ctx).CheckOk();
             }
         }
@@ -173,7 +171,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success, context.GetReceivedCloseOnServerCancelled()));
                 Native.grpcsharp_call_start_serverside(this, ctx).CheckOk();
             }
         }
@@ -183,7 +181,7 @@ namespace Grpc.Core.Internal
             using (completionQueue.NewScope())
             {
                 var ctx = BatchContextSafeHandle.Create();
-                completionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
+                completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, (success, context) => callback(success));
                 Native.grpcsharp_call_send_initial_metadata(this, ctx, metadataArray).CheckOk();
             }
         }
diff --git a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
index 1dbd1f4e34cb7bac20513017ce4ef3de8d71dda0..62864dff0cb88cc774471aed241cfa449b0599ad 100644
--- a/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ChannelSafeHandle.cs
@@ -63,7 +63,7 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_secure_channel_create(credentials, target, channelArgs);
         }
 
-        public CallSafeHandle CreateCall(CompletionRegistry registry, CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
+        public CallSafeHandle CreateCall(CallSafeHandle parentCall, ContextPropagationFlags propagationMask, CompletionQueueSafeHandle cq, string method, string host, Timespec deadline, CallCredentialsSafeHandle credentials)
         {
             using (Profilers.ForCurrentThread().NewScope("ChannelSafeHandle.CreateCall"))
             {
@@ -72,7 +72,7 @@ namespace Grpc.Core.Internal
                 {
                     result.SetCredentials(credentials);
                 }
-                result.Initialize(registry, cq);
+                result.Initialize(cq);
                 return result;
             }
         }
@@ -82,11 +82,10 @@ namespace Grpc.Core.Internal
             return Native.grpcsharp_channel_check_connectivity_state(this, tryToConnect ? 1 : 0);
         }
 
-        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq,
-            CompletionRegistry completionRegistry, BatchCompletionDelegate callback)
+        public void WatchConnectivityState(ChannelState lastObservedState, Timespec deadline, CompletionQueueSafeHandle cq, BatchCompletionDelegate callback)
         {
             var ctx = BatchContextSafeHandle.Create();
-            completionRegistry.RegisterBatchCompletion(ctx, callback);
+            cq.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
             Native.grpcsharp_channel_watch_connectivity_state(this, lastObservedState, deadline, cq, ctx);
         }
 
diff --git a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
index 013f00ff6fc87c3b109c13fb7ece6d3e52e095f7..924de028f513c37e7123ee555c626aa2461eb0af 100644
--- a/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ClientRequestStream.cs
@@ -50,16 +50,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TRequest message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task CompleteAsync()
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendCloseFromClient(taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendCloseFromClientAsync();
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
new file mode 100644
index 0000000000000000000000000000000000000000..5727e3f11f70ed7ddc4d4ef8d7be992d0fdab9b5
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClientSideStatus.cs
@@ -0,0 +1,70 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+// 
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+using System;
+using Grpc.Core;
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// Status + metadata received on client side when call finishes.
+    /// (when receive_status_on_client operation finishes).
+    /// </summary>
+    internal struct ClientSideStatus
+    {
+        readonly Status status;
+        readonly Metadata trailers;
+
+        public ClientSideStatus(Status status, Metadata trailers)
+        {
+            this.status = status;
+            this.trailers = trailers;
+        }
+
+        public Status Status
+        {
+            get
+            {
+                return this.status;
+            }
+        }
+
+        public Metadata Trailers
+        {
+            get
+            {
+                return this.trailers;
+            }
+        }
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/ClockType.cs b/src/csharp/Grpc.Core/Internal/ClockType.cs
new file mode 100644
index 0000000000000000000000000000000000000000..57533c9d2f5550eedc6874e097b859d17ff2fcaf
--- /dev/null
+++ b/src/csharp/Grpc.Core/Internal/ClockType.cs
@@ -0,0 +1,53 @@
+#region Copyright notice and license
+
+// Copyright 2015, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#endregion
+
+namespace Grpc.Core.Internal
+{
+    /// <summary>
+    /// gpr_clock_type from grpc/support/time.h
+    /// </summary>
+    internal enum ClockType
+    {
+        /* Monotonic clock */
+        Monotonic,
+
+        /* Realtime clock */
+        Realtime,
+
+        /* Precise clock good for performance profiling. */
+        Precise,
+
+        /* Timespan - the distance between two time points */
+        Timespan
+    }
+}
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
index 288680792a6eef66aed0477403137cf672f85975..a78e9b70f38c7beb718d9f3bf3bf57d465355137 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueEvent.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
     {
         static readonly NativeMethods Native = NativeMethods.Get();
 
-        public GRPCCompletionType type;
+        public CompletionType type;
         public int success;
         public IntPtr tag;
 
@@ -55,5 +55,20 @@ namespace Grpc.Core.Internal
                 return Native.grpcsharp_sizeof_grpc_event();
             }
         }
+
+        /// <summary>
+        /// grpc_completion_type from grpc/grpc.h
+        /// </summary>
+        internal enum CompletionType
+        {
+            /* Shutting down */
+            Shutdown, 
+
+            /* No event before timeout */
+            Timeout,  
+
+            /* operation completion */
+            OpComplete
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
index 91364cdc70a0b249d476f45eb4ac0d6226e541a0..46f562422315ef5737281dff18c4528de9089e63 100644
--- a/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/CompletionQueueSafeHandle.cs
@@ -45,6 +45,7 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
 
         AtomicCounter shutdownRefcount = new AtomicCounter(1);
+        CompletionRegistry completionRegistry;
 
         private CompletionQueueSafeHandle()
         {
@@ -53,7 +54,13 @@ namespace Grpc.Core.Internal
         public static CompletionQueueSafeHandle Create()
         {
             return Native.grpcsharp_completion_queue_create();
+        }
 
+        public static CompletionQueueSafeHandle Create(CompletionRegistry completionRegistry)
+        {
+            var cq = Native.grpcsharp_completion_queue_create();
+            cq.completionRegistry = completionRegistry;
+            return cq;
         }
 
         public CompletionQueueEvent Next()
@@ -83,6 +90,15 @@ namespace Grpc.Core.Internal
             DecrementShutdownRefcount();
         }
 
+        /// <summary>
+        /// Completion registry associated with this completion queue.
+        /// Doesn't need to be set if only using Pluck() operations.
+        /// </summary>
+        public CompletionRegistry CompletionRegistry
+        {
+            get { return completionRegistry; }
+        }
+
         protected override bool ReleaseHandle()
         {
             Native.grpcsharp_completion_queue_destroy(handle);
diff --git a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
index 4b7124ee749169bfab8dc5c6fdca9ba44b3a80d7..4de543bef7a96625f301e7a0e344547c3884fa90 100644
--- a/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
+++ b/src/csharp/Grpc.Core/Internal/GrpcThreadPool.cs
@@ -33,15 +33,15 @@
 
 using System;
 using System.Collections.Generic;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading;
-using System.Threading.Tasks;
 using Grpc.Core.Logging;
+using Grpc.Core.Utils;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// Pool of threads polling on the same completion queue.
+    /// Pool of threads polling on a set of completions queues.
     /// </summary>
     internal class GrpcThreadPool
     {
@@ -51,25 +51,31 @@ namespace Grpc.Core.Internal
         readonly object myLock = new object();
         readonly List<Thread> threads = new List<Thread>();
         readonly int poolSize;
+        readonly int completionQueueCount;
 
-        CompletionQueueSafeHandle cq;
+        IReadOnlyCollection<CompletionQueueSafeHandle> completionQueues;
 
-        public GrpcThreadPool(GrpcEnvironment environment, int poolSize)
+        /// <summary>
+        /// Creates a thread pool threads polling on a set of completions queues.
+        /// </summary>
+        /// <param name="environment">Environment.</param>
+        /// <param name="poolSize">Pool size.</param>
+        /// <param name="completionQueueCount">Completion queue count.</param>
+        public GrpcThreadPool(GrpcEnvironment environment, int poolSize, int completionQueueCount)
         {
             this.environment = environment;
             this.poolSize = poolSize;
+            this.completionQueueCount = completionQueueCount;
+            GrpcPreconditions.CheckArgument(poolSize >= completionQueueCount,
+                "Thread pool size cannot be smaller than the number of completion queues used.");
         }
 
         public void Start()
         {
             lock (myLock)
             {
-                if (cq != null)
-                {
-                    throw new InvalidOperationException("Already started.");
-                }
-
-                cq = CompletionQueueSafeHandle.Create();
+                GrpcPreconditions.CheckState(completionQueues == null, "Already started.");
+                completionQueues = CreateCompletionQueueList(environment, completionQueueCount);
 
                 for (int i = 0; i < poolSize; i++)
                 {
@@ -82,49 +88,60 @@ namespace Grpc.Core.Internal
         {
             lock (myLock)
             {
-                cq.Shutdown();
+                foreach (var cq in completionQueues)
+                {
+                    cq.Shutdown();
+                }
+
                 foreach (var thread in threads)
                 {
                     thread.Join();
                 }
 
-                cq.Dispose();
+                foreach (var cq in completionQueues)
+                {
+                    cq.Dispose();
+                }
             }
         }
 
-        internal CompletionQueueSafeHandle CompletionQueue
+        internal IReadOnlyCollection<CompletionQueueSafeHandle> CompletionQueues
         {
             get
             {
-                return cq;
+                return completionQueues;
             }
         }
 
-        private Thread CreateAndStartThread(int i)
+        private Thread CreateAndStartThread(int threadIndex)
         {
-            var thread = new Thread(new ThreadStart(RunHandlerLoop));
+            var cqIndex = threadIndex % completionQueues.Count;
+            var cq = completionQueues.ElementAt(cqIndex);
+
+            var thread = new Thread(new ThreadStart(() => RunHandlerLoop(cq)));
             thread.IsBackground = false;
+            thread.Name = string.Format("grpc {0} (cq {1})", threadIndex, cqIndex);
             thread.Start();
-            thread.Name = "grpc " + i;
+
             return thread;
         }
 
         /// <summary>
         /// Body of the polling thread.
         /// </summary>
-        private void RunHandlerLoop()
+        private void RunHandlerLoop(CompletionQueueSafeHandle cq)
         {
             CompletionQueueEvent ev;
             do
             {
                 ev = cq.Next();
-                if (ev.type == GRPCCompletionType.OpComplete)
+                if (ev.type == CompletionQueueEvent.CompletionType.OpComplete)
                 {
                     bool success = (ev.success != 0);
                     IntPtr tag = ev.tag;
                     try
                     {
-                        var callback = environment.CompletionRegistry.Extract(tag);
+                        var callback = cq.CompletionRegistry.Extract(tag);
                         callback(success);
                     }
                     catch (Exception e)
@@ -133,7 +150,18 @@ namespace Grpc.Core.Internal
                     }
                 }
             }
-            while (ev.type != GRPCCompletionType.Shutdown);
+            while (ev.type != CompletionQueueEvent.CompletionType.Shutdown);
+        }
+
+        private static IReadOnlyCollection<CompletionQueueSafeHandle> CreateCompletionQueueList(GrpcEnvironment environment, int completionQueueCount)
+        {
+            var list = new List<CompletionQueueSafeHandle>();
+            for (int i = 0; i < completionQueueCount; i++)
+            {
+                var completionRegistry = new CompletionRegistry(environment);
+                list.Add(CompletionQueueSafeHandle.Create(completionRegistry));
+            }
+            return list.AsReadOnly();
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
index 25735d5262d6335492f3881e1fb4ed18cfac0aca..dc9f62fdab42e2b58c60d6c4b91b0941a9255c5e 100644
--- a/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/MetadataArraySafeHandle.cs
@@ -50,6 +50,11 @@ namespace Grpc.Core.Internal
         {
             using (Profilers.ForCurrentThread().NewScope("MetadataArraySafeHandle.Create"))
             {
+                if (metadata.Count == 0)
+                {
+                    return new MetadataArraySafeHandle();
+                }
+
                 // TODO(jtattermusch): we might wanna check that the metadata is readonly 
                 var metadataArray = Native.grpcsharp_metadata_array_create(new UIntPtr((ulong)metadata.Count));
                 for (int i = 0; i < metadata.Count; i++)
diff --git a/src/csharp/Grpc.Core/Internal/NativeMethods.cs b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
index c277c73ef04ed92c818e7c786722d224124fc82c..65607ed12042e74760df19a70e5be7db7b2dfc61 100644
--- a/src/csharp/Grpc.Core/Internal/NativeMethods.cs
+++ b/src/csharp/Grpc.Core/Internal/NativeMethods.cs
@@ -137,6 +137,7 @@ namespace Grpc.Core.Internal
         public readonly Delegates.grpcsharp_server_credentials_release_delegate grpcsharp_server_credentials_release;
 
         public readonly Delegates.grpcsharp_server_create_delegate grpcsharp_server_create;
+        public readonly Delegates.grpcsharp_server_register_completion_queue_delegate grpcsharp_server_register_completion_queue;
         public readonly Delegates.grpcsharp_server_add_insecure_http2_port_delegate grpcsharp_server_add_insecure_http2_port;
         public readonly Delegates.grpcsharp_server_add_secure_http2_port_delegate grpcsharp_server_add_secure_http2_port;
         public readonly Delegates.grpcsharp_server_start_delegate grpcsharp_server_start;
@@ -244,6 +245,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = GetMethodDelegate<Delegates.grpcsharp_server_credentials_release_delegate>(library);
 
                 this.grpcsharp_server_create = GetMethodDelegate<Delegates.grpcsharp_server_create_delegate>(library);
+                this.grpcsharp_server_register_completion_queue = GetMethodDelegate<Delegates.grpcsharp_server_register_completion_queue_delegate>(library);
                 this.grpcsharp_server_add_insecure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_insecure_http2_port_delegate>(library);
                 this.grpcsharp_server_add_secure_http2_port = GetMethodDelegate<Delegates.grpcsharp_server_add_secure_http2_port_delegate>(library);
                 this.grpcsharp_server_start = GetMethodDelegate<Delegates.grpcsharp_server_start_delegate>(library);
@@ -348,6 +350,7 @@ namespace Grpc.Core.Internal
                 this.grpcsharp_server_credentials_release = PInvokeMethods.grpcsharp_server_credentials_release;
 
                 this.grpcsharp_server_create = PInvokeMethods.grpcsharp_server_create;
+                this.grpcsharp_server_register_completion_queue = PInvokeMethods.grpcsharp_server_register_completion_queue;
                 this.grpcsharp_server_add_insecure_http2_port = PInvokeMethods.grpcsharp_server_add_insecure_http2_port;
                 this.grpcsharp_server_add_secure_http2_port = PInvokeMethods.grpcsharp_server_add_secure_http2_port;
                 this.grpcsharp_server_start = PInvokeMethods.grpcsharp_server_start;
@@ -418,33 +421,33 @@ namespace Grpc.Core.Internal
             public delegate CallCredentialsSafeHandle grpcsharp_composite_call_credentials_create_delegate(CallCredentialsSafeHandle creds1, CallCredentialsSafeHandle creds2);
             public delegate void grpcsharp_call_credentials_release_delegate(IntPtr credentials);
 
-            public delegate GRPCCallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
-            public delegate GRPCCallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
-            public delegate GRPCCallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_cancel_delegate(CallSafeHandle call);
+            public delegate CallError grpcsharp_call_cancel_with_status_delegate(CallSafeHandle call, StatusCode status, string description);
+            public delegate CallError grpcsharp_call_start_unary_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_client_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_server_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_duplex_streaming_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
-            public delegate GRPCCallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_close_from_client_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_status_from_server_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
-            public delegate GRPCCallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_message_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_recv_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_start_serverside_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
-            public delegate GRPCCallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
+            public delegate CallError grpcsharp_call_send_initial_metadata_delegate(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
-            public delegate GRPCCallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public delegate CallError grpcsharp_call_set_credentials_delegate(CallSafeHandle call, CallCredentialsSafeHandle credentials);
             public delegate CStringSafeHandle grpcsharp_call_get_peer_delegate(CallSafeHandle call);
             public delegate void grpcsharp_call_destroy_delegate(IntPtr call);
 
@@ -493,23 +496,24 @@ namespace Grpc.Core.Internal
             public delegate ServerCredentialsSafeHandle grpcsharp_ssl_server_credentials_create_delegate(string pemRootCerts, string[] keyCertPairCertChainArray, string[] keyCertPairPrivateKeyArray, UIntPtr numKeyCertPairs, bool forceClientAuth);
             public delegate void grpcsharp_server_credentials_release_delegate(IntPtr credentials);
 
-            public delegate ServerSafeHandle grpcsharp_server_create_delegate(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public delegate ServerSafeHandle grpcsharp_server_create_delegate(ChannelArgsSafeHandle args);
+            public delegate void grpcsharp_server_register_completion_queue_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq);
             public delegate int grpcsharp_server_add_insecure_http2_port_delegate(ServerSafeHandle server, string addr);
             public delegate int grpcsharp_server_add_secure_http2_port_delegate(ServerSafeHandle server, string addr, ServerCredentialsSafeHandle creds);
             public delegate void grpcsharp_server_start_delegate(ServerSafeHandle server);
-            public delegate GRPCCallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public delegate CallError grpcsharp_server_request_call_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_cancel_all_calls_delegate(ServerSafeHandle server);
             public delegate void grpcsharp_server_shutdown_and_notify_callback_delegate(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
             public delegate void grpcsharp_server_destroy_delegate(IntPtr server);
 
-            public delegate Timespec gprsharp_now_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_future_delegate(GPRClockType clockType);
-            public delegate Timespec gprsharp_inf_past_delegate(GPRClockType clockType);
+            public delegate Timespec gprsharp_now_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_future_delegate(ClockType clockType);
+            public delegate Timespec gprsharp_inf_past_delegate(ClockType clockType);
 
-            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, GPRClockType targetClock);
+            public delegate Timespec gprsharp_convert_clock_type_delegate(Timespec t, ClockType targetClock);
             public delegate int gprsharp_sizeof_timespec_delegate();
 
-            public delegate GRPCCallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public delegate CallError grpcsharp_test_callback_delegate([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
             public delegate IntPtr grpcsharp_test_nop_delegate(IntPtr ptr);
         }
 
@@ -587,59 +591,59 @@ namespace Grpc.Core.Internal
             // CallSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel(CallSafeHandle call);
+            public static extern CallError grpcsharp_call_cancel(CallSafeHandle call);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
+            public static extern CallError grpcsharp_call_cancel_with_status(CallSafeHandle call, StatusCode status, string description);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_unary(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_unary(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_client_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_server_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen,
                 MetadataArraySafeHandle metadataArray, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_duplex_streaming(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx, byte[] sendBuffer, UIntPtr sendBufferLen, WriteFlags writeFlags, bool sendEmptyInitialMetadata);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_close_from_client(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_status_from_server(CallSafeHandle call,
                 BatchContextSafeHandle ctx, StatusCode statusCode, string statusMessage, MetadataArraySafeHandle metadataArray, bool sendEmptyInitialMetadata,
                 byte[] optionalSendBuffer, UIntPtr optionalSendBufferLen, WriteFlags writeFlags);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_message(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_message(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_recv_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_start_serverside(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_start_serverside(CallSafeHandle call,
                 BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
+            public static extern CallError grpcsharp_call_send_initial_metadata(CallSafeHandle call,
                 BatchContextSafeHandle ctx, MetadataArraySafeHandle metadataArray);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
+            public static extern CallError grpcsharp_call_set_credentials(CallSafeHandle call, CallCredentialsSafeHandle credentials);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern CStringSafeHandle grpcsharp_call_get_peer(CallSafeHandle call);
@@ -773,7 +777,10 @@ namespace Grpc.Core.Internal
             // ServerSafeHandle
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern ServerSafeHandle grpcsharp_server_create(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args);
+            public static extern ServerSafeHandle grpcsharp_server_create(ChannelArgsSafeHandle args);
+
+            [DllImport("grpc_csharp_ext.dll")]
+            public static extern void grpcsharp_server_register_completion_queue(ServerSafeHandle server, CompletionQueueSafeHandle cq);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int grpcsharp_server_add_insecure_http2_port(ServerSafeHandle server, string addr);
@@ -785,7 +792,7 @@ namespace Grpc.Core.Internal
             public static extern void grpcsharp_server_start(ServerSafeHandle server);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
+            public static extern CallError grpcsharp_server_request_call(ServerSafeHandle server, CompletionQueueSafeHandle cq, BatchContextSafeHandle ctx);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern void grpcsharp_server_cancel_all_calls(ServerSafeHandle server);
@@ -799,16 +806,16 @@ namespace Grpc.Core.Internal
             // Timespec
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_now(GPRClockType clockType);
+            public static extern Timespec gprsharp_now(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_future(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_future(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_inf_past(GPRClockType clockType);
+            public static extern Timespec gprsharp_inf_past(ClockType clockType);
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern Timespec gprsharp_convert_clock_type(Timespec t, GPRClockType targetClock);
+            public static extern Timespec gprsharp_convert_clock_type(Timespec t, ClockType targetClock);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern int gprsharp_sizeof_timespec();
@@ -816,7 +823,7 @@ namespace Grpc.Core.Internal
             // Testing
 
             [DllImport("grpc_csharp_ext.dll")]
-            public static extern GRPCCallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
+            public static extern CallError grpcsharp_test_callback([MarshalAs(UnmanagedType.FunctionPtr)] OpCompletionDelegate callback);
 
             [DllImport("grpc_csharp_ext.dll")]
             public static extern IntPtr grpcsharp_test_nop(IntPtr ptr);
diff --git a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
index 85b7a4b01e90067ad4d435cd47ff5cfd3020671d..6a2f52016361cef5469b4b389e4fe36e99ae040d 100644
--- a/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerCallHandler.cs
@@ -44,7 +44,7 @@ namespace Grpc.Core.Internal
 {
     internal interface IServerCallHandler
     {
-        Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment);
+        Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq);
     }
 
     internal class UnaryServerCallHandler<TRequest, TResponse> : IServerCallHandler
@@ -62,14 +62,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -121,14 +121,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -179,14 +179,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -237,14 +237,14 @@ namespace Grpc.Core.Internal
             this.handler = handler;
         }
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             var asyncCall = new AsyncCallServer<TRequest, TResponse>(
                 method.ResponseMarshaller.Serializer,
                 method.RequestMarshaller.Deserializer,
-                environment, newRpc.Server);
+                newRpc.Server);
 
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             var requestStream = new ServerRequestStream<TRequest, TResponse>(asyncCall);
             var responseStream = new ServerResponseStream<TRequest, TResponse>(asyncCall);
@@ -281,13 +281,13 @@ namespace Grpc.Core.Internal
     {
         public static readonly NoSuchMethodCallHandler Instance = new NoSuchMethodCallHandler();
 
-        public async Task HandleCall(ServerRpcNew newRpc, GrpcEnvironment environment)
+        public async Task HandleCall(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             // We don't care about the payload type here.
             var asyncCall = new AsyncCallServer<byte[], byte[]>(
-                (payload) => payload, (payload) => payload, environment, newRpc.Server);
+                (payload) => payload, (payload) => payload, newRpc.Server);
             
-            asyncCall.Initialize(newRpc.Call);
+            asyncCall.Initialize(newRpc.Call, cq);
             var finishedTask = asyncCall.ServerSideCallAsync();
             await asyncCall.SendStatusFromServerAsync(new Status(StatusCode.Unimplemented, ""), Metadata.Empty, null).ConfigureAwait(false);
             await finishedTask.ConfigureAwait(false);
@@ -317,7 +317,7 @@ namespace Grpc.Core.Internal
             where TRequest : class
             where TResponse : class
         {
-            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(GPRClockType.Realtime).ToDateTime();
+            DateTime realtimeDeadline = newRpc.Deadline.ToClockType(ClockType.Realtime).ToDateTime();
 
             return new ServerCallContext(newRpc.Call, newRpc.Method, newRpc.Host, peer, realtimeDeadline,
                 newRpc.RequestMetadata, cancellationToken, serverResponseStream.WriteResponseHeadersAsync, serverResponseStream);
diff --git a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
index ecfee0bfddbc1ade1a9826397b12b9a5d281a89e..25b79b43988df533c47eddafb9ec27c44ccedc5e 100644
--- a/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerResponseStream.cs
@@ -52,16 +52,12 @@ namespace Grpc.Core.Internal
 
         public Task WriteAsync(TResponse message)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendMessage(message, GetWriteFlags(), taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendMessageAsync(message, GetWriteFlags());
         }
 
         public Task WriteResponseHeadersAsync(Metadata responseHeaders)
         {
-            var taskSource = new AsyncCompletionTaskSource<object>();
-            call.StartSendInitialMetadata(responseHeaders, taskSource.CompletionDelegate);
-            return taskSource.Task;
+            return call.SendInitialMetadataAsync(responseHeaders);
         }
 
         public WriteOptions WriteOptions
diff --git a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
similarity index 60%
rename from src/csharp/Grpc.Core/Internal/AsyncCompletion.cs
rename to src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
index 7e86fddb4d89ef583b49babf78da6f994ae180c0..e4f1880bdb0e6f2da6973c8c1dfd80bdce86ccee 100644
--- a/src/csharp/Grpc.Core/Internal/AsyncCompletion.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerRpcNew.cs
@@ -32,63 +32,78 @@
 #endregion
 
 using System;
-using System.Diagnostics;
-using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
-using System.Threading;
-using System.Threading.Tasks;
-using Grpc.Core.Internal;
-using Grpc.Core.Utils;
+using Grpc.Core;
 
 namespace Grpc.Core.Internal
 {
     /// <summary>
-    /// If error != null, there's been an error or operation has been cancelled.
+    /// Details of a newly received RPC.
     /// </summary>
-    internal delegate void AsyncCompletionDelegate<T>(T result, Exception error);
-
-    /// <summary>
-    /// Helper for transforming AsyncCompletionDelegate into full-fledged Task.
-    /// </summary>
-    internal class AsyncCompletionTaskSource<T>
+    internal struct ServerRpcNew
     {
-        readonly TaskCompletionSource<T> tcs = new TaskCompletionSource<T>();
-        readonly AsyncCompletionDelegate<T> completionDelegate;
+        readonly Server server;
+        readonly CallSafeHandle call;
+        readonly string method;
+        readonly string host;
+        readonly Timespec deadline;
+        readonly Metadata requestMetadata;
+
+        public ServerRpcNew(Server server, CallSafeHandle call, string method, string host, Timespec deadline, Metadata requestMetadata)
+        {
+            this.server = server;
+            this.call = call;
+            this.method = method;
+            this.host = host;
+            this.deadline = deadline;
+            this.requestMetadata = requestMetadata;
+        }
+
+        public Server Server
+        {
+            get
+            {
+                return this.server;
+            }
+        }
 
-        public AsyncCompletionTaskSource()
+        public CallSafeHandle Call
         {
-            completionDelegate = new AsyncCompletionDelegate<T>(HandleCompletion);
+            get
+            {
+                return this.call;
+            }
         }
 
-        public Task<T> Task
+        public string Method
         {
             get
             {
-                return tcs.Task;
+                return this.method;
             }
         }
 
-        public AsyncCompletionDelegate<T> CompletionDelegate
+        public string Host
         {
             get
             {
-                return completionDelegate;
+                return this.host;
             }
         }
 
-        private void HandleCompletion(T value, Exception error)
+        public Timespec Deadline
         {
-            if (error == null)
+            get
             {
-                tcs.SetResult(value);
-                return;
+                return this.deadline;
             }
-            if (error is OperationCanceledException)
+        }
+
+        public Metadata RequestMetadata
+        {
+            get
             {
-                tcs.SetCanceled();
-                return;
+                return this.requestMetadata;
             }
-            tcs.SetException(error);
         }
     }
 }
diff --git a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
index 6b5f70e220737eea2c4f26d22b88773ddc19d7b4..85813027064f47359be46b25bdb3bf8cf5e0736e 100644
--- a/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
+++ b/src/csharp/Grpc.Core/Internal/ServerSafeHandle.cs
@@ -31,12 +31,6 @@
 
 #endregion
 
-using System;
-using System.Collections.Concurrent;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-using Grpc.Core.Utils;
-
 namespace Grpc.Core.Internal
 {
     /// <summary>
@@ -50,12 +44,17 @@ namespace Grpc.Core.Internal
         {
         }
 
-        public static ServerSafeHandle NewServer(CompletionQueueSafeHandle cq, ChannelArgsSafeHandle args)
+        public static ServerSafeHandle NewServer(ChannelArgsSafeHandle args)
         {
             // Increment reference count for the native gRPC environment to make sure we don't do grpc_shutdown() before destroying the server handle.
             // Doing so would make object finalizer crash if we end up abandoning the handle.
             GrpcEnvironment.GrpcNativeInit();
-            return Native.grpcsharp_server_create(cq, args);
+            return Native.grpcsharp_server_create(args);
+        }
+
+        public void RegisterCompletionQueue(CompletionQueueSafeHandle cq)
+        {
+            Native.grpcsharp_server_register_completion_queue(this, cq);
         }
 
         public int AddInsecurePort(string addr)
@@ -73,18 +72,18 @@ namespace Grpc.Core.Internal
             Native.grpcsharp_server_start(this);
         }
     
-        public void ShutdownAndNotify(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void ShutdownAndNotify(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
             var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_shutdown_and_notify_callback(this, environment.CompletionQueue, ctx);
+            completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+            Native.grpcsharp_server_shutdown_and_notify_callback(this, completionQueue, ctx);
         }
 
-        public void RequestCall(BatchCompletionDelegate callback, GrpcEnvironment environment)
+        public void RequestCall(BatchCompletionDelegate callback, CompletionQueueSafeHandle completionQueue)
         {
             var ctx = BatchContextSafeHandle.Create();
-            environment.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
-            Native.grpcsharp_server_request_call(this, environment.CompletionQueue, ctx).CheckOk();
+            completionQueue.CompletionRegistry.RegisterBatchCompletion(ctx, callback);
+            Native.grpcsharp_server_request_call(this, completionQueue, ctx).CheckOk();
         }
 
         protected override bool ReleaseHandle()
diff --git a/src/csharp/Grpc.Core/Internal/Timespec.cs b/src/csharp/Grpc.Core/Internal/Timespec.cs
index 56172a5dda473b87b83e5ab13a5a53a05ae1d71b..c9fd710e1e8dd2489a065deb290d44d580e5f7c8 100644
--- a/src/csharp/Grpc.Core/Internal/Timespec.cs
+++ b/src/csharp/Grpc.Core/Internal/Timespec.cs
@@ -49,11 +49,11 @@ namespace Grpc.Core.Internal
         static readonly NativeMethods Native = NativeMethods.Get();
         static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
 
-        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, GPRClockType.Realtime)
+        public Timespec(long tv_sec, int tv_nsec) : this(tv_sec, tv_nsec, ClockType.Realtime)
         {
         }
 
-        public Timespec(long tv_sec, int tv_nsec, GPRClockType clock_type)
+        public Timespec(long tv_sec, int tv_nsec, ClockType clock_type)
         {
             this.tv_sec = tv_sec;
             this.tv_nsec = tv_nsec;
@@ -62,7 +62,7 @@ namespace Grpc.Core.Internal
 
         private long tv_sec;
         private int tv_nsec;
-        private GPRClockType clock_type;
+        private ClockType clock_type;
 
         /// <summary>
         /// Timespec a long time in the future.
@@ -71,7 +71,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_future(GPRClockType.Realtime);
+                return new Timespec(long.MaxValue, 0, ClockType.Realtime);
             }
         }
 
@@ -82,7 +82,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_inf_past(GPRClockType.Realtime);
+                return new Timespec(long.MinValue, 0, ClockType.Realtime);
             }
         }
 
@@ -93,7 +93,7 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Realtime);
+                return Native.gprsharp_now(ClockType.Realtime);
             }
         }
 
@@ -122,7 +122,7 @@ namespace Grpc.Core.Internal
         /// <summary>
         /// Converts the timespec to desired clock type.
         /// </summary>
-        public Timespec ToClockType(GPRClockType targetClock)
+        public Timespec ToClockType(ClockType targetClock)
         {
             return Native.gprsharp_convert_clock_type(this, targetClock);
         }
@@ -142,7 +142,7 @@ namespace Grpc.Core.Internal
         public DateTime ToDateTime()
         {
             GrpcPreconditions.CheckState(tv_nsec >= 0 && tv_nsec < NanosPerSecond);
-            GrpcPreconditions.CheckState(clock_type == GPRClockType.Realtime);
+            GrpcPreconditions.CheckState(clock_type == ClockType.Realtime);
 
             // fast path for InfFuture
             if (this.Equals(InfFuture))
@@ -227,10 +227,11 @@ namespace Grpc.Core.Internal
         {
             get
             {
-                return Native.gprsharp_now(GPRClockType.Precise);
+                return Native.gprsharp_now(ClockType.Precise);
             }
         }
 
+        // for tests only
         internal static int NativeSize
         {
             get
@@ -238,5 +239,23 @@ namespace Grpc.Core.Internal
                 return Native.gprsharp_sizeof_timespec();
             }
         }
+
+        // for tests only
+        internal static Timespec NativeInfFuture
+        {
+            get
+            {
+                return Native.gprsharp_inf_future(ClockType.Realtime);
+            }
+        }
+
+        // for tests only
+        public static Timespec NativeInfPast
+        {
+            get
+            {
+                return Native.gprsharp_inf_past(ClockType.Realtime);
+            }
+        }
     }
 }
diff --git a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
index bde74945fbf691828fff12e80d7c434610747f43..370fa986874fcc01778feeca7dc05da47250fd0d 100644
--- a/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
+++ b/src/csharp/Grpc.Core/Properties/AssemblyInfo.cs
@@ -16,6 +16,12 @@ using System.Runtime.CompilerServices;
     "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
     "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
     "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting,PublicKey=" +
+    "00240000048000009400000006020000002400005253413100040000010001002f5797a92c6fcde81bd4098f43" +
+    "0442bb8e12768722de0b0cb1b15e955b32a11352740ee59f2c94c48edc8e177d1052536b8ac651bce11ce5da3a" +
+    "27fc95aff3dc604a6971417453f9483c7b5e836756d5b271bf8f2403fe186e31956148c03d804487cf642f8cc0" +
+    "71394ee9672dfe5b55ea0f95dfd5a7f77d22c962ccf51320d3")]
 #else
 [assembly: InternalsVisibleTo("Grpc.Core.Tests")]
+[assembly: InternalsVisibleTo("Grpc.IntegrationTesting")]
 #endif
diff --git a/src/csharp/Grpc.Core/Server.cs b/src/csharp/Grpc.Core/Server.cs
index d538a4671f6dcafdc6f7086b13e040ee70517972..069185e13af9b6da2af5fee9ecaea1174d951d3f 100644
--- a/src/csharp/Grpc.Core/Server.cs
+++ b/src/csharp/Grpc.Core/Server.cs
@@ -34,8 +34,7 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
+using System.Linq;
 using System.Threading.Tasks;
 using Grpc.Core.Internal;
 using Grpc.Core.Logging;
@@ -48,7 +47,7 @@ namespace Grpc.Core
     /// </summary>
     public class Server
     {
-        const int InitialAllowRpcTokenCount = 10;
+        const int InitialAllowRpcTokenCountPerCq = 10;
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<Server>();
 
         readonly AtomicCounter activeCallCounter = new AtomicCounter();
@@ -80,7 +79,12 @@ namespace Grpc.Core
             this.options = options != null ? new List<ChannelOption>(options) : new List<ChannelOption>();
             using (var channelArgs = ChannelOptions.CreateChannelArgs(this.options))
             {
-                this.handle = ServerSafeHandle.NewServer(environment.CompletionQueue, channelArgs);
+                this.handle = ServerSafeHandle.NewServer(channelArgs);
+            }
+
+            foreach (var cq in environment.CompletionQueues)
+            {
+                this.handle.RegisterCompletionQueue(cq);
             }
         }
 
@@ -133,9 +137,12 @@ namespace Grpc.Core
 
                 // Starting with more than one AllowOneRpc tokens can significantly increase
                 // unary RPC throughput.
-                for (int i = 0; i < InitialAllowRpcTokenCount; i++)
+                for (int i = 0; i < InitialAllowRpcTokenCountPerCq; i++)
                 {
-                    AllowOneRpc();
+                    foreach (var cq in environment.CompletionQueues)
+                    {
+                        AllowOneRpc(cq);
+                    }
                 }
             }
         }
@@ -154,7 +161,8 @@ namespace Grpc.Core
                 shutdownRequested = true;
             }
 
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
+            var cq = environment.CompletionQueues.First();  // any cq will do
+            handle.ShutdownAndNotify(HandleServerShutdown, cq);
             await shutdownTcs.Task.ConfigureAwait(false);
             DisposeHandle();
 
@@ -174,7 +182,8 @@ namespace Grpc.Core
                 shutdownRequested = true;
             }
 
-            handle.ShutdownAndNotify(HandleServerShutdown, environment);
+            var cq = environment.CompletionQueues.First();  // any cq will do
+            handle.ShutdownAndNotify(HandleServerShutdown, cq);
             handle.CancelAllCalls();
             await shutdownTcs.Task.ConfigureAwait(false);
             DisposeHandle();
@@ -244,11 +253,11 @@ namespace Grpc.Core
         /// <summary>
         /// Allows one new RPC call to be received by server.
         /// </summary>
-        private void AllowOneRpc()
+        private void AllowOneRpc(CompletionQueueSafeHandle cq)
         {
             if (!shutdownRequested)
             {
-                handle.RequestCall(HandleNewServerRpc, environment);
+                handle.RequestCall((success, ctx) => HandleNewServerRpc(success, ctx, cq), cq);
             }
         }
 
@@ -265,7 +274,7 @@ namespace Grpc.Core
         /// <summary>
         /// Selects corresponding handler for given call and handles the call.
         /// </summary>
-        private async Task HandleCallAsync(ServerRpcNew newRpc)
+        private async Task HandleCallAsync(ServerRpcNew newRpc, CompletionQueueSafeHandle cq)
         {
             try
             {
@@ -274,7 +283,7 @@ namespace Grpc.Core
                 {
                     callHandler = NoSuchMethodCallHandler.Instance;
                 }
-                await callHandler.HandleCall(newRpc, environment).ConfigureAwait(false);
+                await callHandler.HandleCall(newRpc, cq).ConfigureAwait(false);
             }
             catch (Exception e)
             {
@@ -285,9 +294,9 @@ namespace Grpc.Core
         /// <summary>
         /// Handles the native callback.
         /// </summary>
-        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx)
+        private void HandleNewServerRpc(bool success, BatchContextSafeHandle ctx, CompletionQueueSafeHandle cq)
         {
-			Task.Run(() => AllowOneRpc());
+			Task.Run(() => AllowOneRpc(cq));
 
             if (success)
             {
@@ -296,7 +305,7 @@ namespace Grpc.Core
                 // after server shutdown, the callback returns with null call
                 if (!newRpc.Call.IsInvalid)
                 {
-                    HandleCallAsync(newRpc);  // we don't need to await.
+                    HandleCallAsync(newRpc, cq);  // we don't need to await.
                 }
             }
         }
diff --git a/src/csharp/Grpc.Core/WriteOptions.cs b/src/csharp/Grpc.Core/WriteOptions.cs
index 7523ada84a8ee33b1af4a6e1bcd82ce9069a7083..4c9706d966f608427f7b4f96ffac566fcac60779 100644
--- a/src/csharp/Grpc.Core/WriteOptions.cs
+++ b/src/csharp/Grpc.Core/WriteOptions.cs
@@ -1,6 +1,6 @@
 #region Copyright notice and license
 
-// Copyright 2015, Google Inc.
+// Copyright 2015-2016, Google Inc.
 // All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
@@ -64,7 +64,7 @@ namespace Grpc.Core
         /// </summary>
         public static readonly WriteOptions Default = new WriteOptions();
             
-        private WriteFlags flags;
+        private readonly WriteFlags flags;
 
         /// <summary>
         /// Initializes a new instance of <c>WriteOptions</c> class.
diff --git a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
index ed2092500a3ff5666877469478927e72fc2a78da..39b9ae08e64ef99fcfc9c3c83764d65c95558034 100644
--- a/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
+++ b/src/csharp/Grpc.IntegrationTesting/ClientRunners.cs
@@ -32,6 +32,7 @@
 #endregion
 
 using System;
+using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
@@ -41,7 +42,9 @@ using System.Threading;
 using System.Threading.Tasks;
 using Google.Protobuf;
 using Grpc.Core;
+using Grpc.Core.Internal;
 using Grpc.Core.Logging;
+using Grpc.Core.Profiling;
 using Grpc.Core.Utils;
 using NUnit.Framework;
 using Grpc.Testing;
@@ -55,6 +58,15 @@ namespace Grpc.IntegrationTesting
     {
         static readonly ILogger Logger = GrpcEnvironment.Logger.ForType<ClientRunners>();
 
+        // Profilers to use for clients.
+        static readonly BlockingCollection<BasicProfiler> profilers = new BlockingCollection<BasicProfiler>();
+
+        internal static void AddProfiler(BasicProfiler profiler)
+        {
+            GrpcPreconditions.CheckNotNull(profiler);
+            profilers.Add(profiler);
+        }
+
         /// <summary>
         /// Creates a started client runner.
         /// </summary>
@@ -83,7 +95,8 @@ namespace Grpc.IntegrationTesting
                 config.OutstandingRpcsPerChannel,
                 config.LoadParams,
                 config.PayloadConfig,
-                config.HistogramParams);
+                config.HistogramParams,
+                () => GetNextProfiler());
         }
 
         private static List<Channel> CreateChannels(int clientChannels, IEnumerable<string> serverTargets, SecurityParams securityParams)
@@ -110,9 +123,16 @@ namespace Grpc.IntegrationTesting
             }
             return result;
         }
+
+        private static BasicProfiler GetNextProfiler()
+        {
+            BasicProfiler result = null;
+            profilers.TryTake(out result);
+            return result;
+        }
     }
 
-    public class ClientRunnerImpl : IClientRunner
+    internal class ClientRunnerImpl : IClientRunner
     {
         const double SecondsToNanos = 1e9;
 
@@ -125,8 +145,9 @@ namespace Grpc.IntegrationTesting
         readonly List<Task> runnerTasks;
         readonly CancellationTokenSource stoppedCts = new CancellationTokenSource();
         readonly WallClockStopwatch wallClockStopwatch = new WallClockStopwatch();
+        readonly AtomicCounter statsResetCount = new AtomicCounter();
         
-        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams)
+        public ClientRunnerImpl(List<Channel> channels, ClientType clientType, RpcType rpcType, int outstandingRpcsPerChannel, LoadParams loadParams, PayloadConfig payloadConfig, HistogramParams histogramParams, Func<BasicProfiler> profilerFactory)
         {
             GrpcPreconditions.CheckArgument(outstandingRpcsPerChannel > 0, "outstandingRpcsPerChannel");
             GrpcPreconditions.CheckNotNull(histogramParams, "histogramParams");
@@ -142,7 +163,8 @@ namespace Grpc.IntegrationTesting
                 for (int i = 0; i < outstandingRpcsPerChannel; i++)
                 {
                     var timer = CreateTimer(loadParams, 1.0 / this.channels.Count / outstandingRpcsPerChannel);
-                    this.runnerTasks.Add(RunClientAsync(channel, timer));
+                    var optionalProfiler = profilerFactory();
+                    this.runnerTasks.Add(RunClientAsync(channel, timer, optionalProfiler));
                 }
             }
         }
@@ -152,6 +174,11 @@ namespace Grpc.IntegrationTesting
             var histogramData = histogram.GetSnapshot(reset);
             var secondsElapsed = wallClockStopwatch.GetElapsedSnapshot(reset).TotalSeconds;
 
+            if (reset)
+            {
+                statsResetCount.Increment();
+            }
+
             // TODO: populate user time and system time
             return new ClientStats
             {
@@ -175,14 +202,28 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private void RunUnary(Channel channel, IInterarrivalTimer timer)
+        private void RunUnary(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
+            if (optionalProfiler != null)
+            {
+                Profilers.SetForCurrentThread(optionalProfiler);
+            }
+
+            bool profilerReset = false;
+
             var client = BenchmarkService.NewClient(channel);
             var request = CreateSimpleRequest();
             var stopwatch = new Stopwatch();
 
             while (!stoppedCts.Token.IsCancellationRequested)
             {
+                // after the first stats reset, also reset the profiler.
+                if (optionalProfiler != null && !profilerReset && statsResetCount.Count > 0)
+                {
+                    optionalProfiler.Reset();
+                    profilerReset = true;
+                }
+
                 stopwatch.Restart();
                 client.UnaryCall(request);
                 stopwatch.Stop();
@@ -268,7 +309,7 @@ namespace Grpc.IntegrationTesting
             }
         }
 
-        private Task RunClientAsync(Channel channel, IInterarrivalTimer timer)
+        private Task RunClientAsync(Channel channel, IInterarrivalTimer timer, BasicProfiler optionalProfiler)
         {
             if (payloadConfig.PayloadCase == PayloadConfig.PayloadOneofCase.BytebufParams)
             {
@@ -282,7 +323,7 @@ namespace Grpc.IntegrationTesting
             {
                 GrpcPreconditions.CheckArgument(rpcType == RpcType.Unary, "Sync client can only be used for Unary calls in C#");
                 // create a dedicated thread for the synchronous client
-                return Task.Factory.StartNew(() => RunUnary(channel, timer), TaskCreationOptions.LongRunning);
+                return Task.Factory.StartNew(() => RunUnary(channel, timer, optionalProfiler), TaskCreationOptions.LongRunning);
             }
             else if (clientType == ClientType.AsyncClient)
             {
diff --git a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
index 1541cfd7bb1acdf2dafdf9356cfe8e262f3b1513..aea40afee2c4fbc5a855fb2d7f697b2f072faf2a 100644
--- a/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
+++ b/src/csharp/Grpc.IntegrationTesting/InteropClient.cs
@@ -471,8 +471,16 @@ namespace Grpc.IntegrationTesting
 
                 cts.Cancel();
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    Assert.AreEqual(StatusCode.Cancelled, ex.Status.StatusCode);
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -497,9 +505,16 @@ namespace Grpc.IntegrationTesting
                     // Deadline was reached before write has started. Eat the exception and continue.
                 }
 
-                var ex = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.MoveNext());
-                // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
-                Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                try
+                {
+                    await call.ResponseStream.MoveNext();
+                    Assert.Fail();
+                }
+                catch (RpcException ex)
+                {
+                    // We can't guarantee the status code always DeadlineExceeded. See issue #2685.
+                    Assert.Contains(ex.Status.StatusCode, new[] { StatusCode.DeadlineExceeded, StatusCode.Internal });
+                }
             }
             Console.WriteLine("Passed!");
         }
@@ -577,9 +592,17 @@ namespace Grpc.IntegrationTesting
                 await call.RequestStream.WriteAsync(request);
                 await call.RequestStream.CompleteAsync();
 
-                var e = Assert.ThrowsAsync<RpcException>(async () => await call.ResponseStream.ToListAsync());
-                Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
-                Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                try
+                {
+                    // cannot use Assert.ThrowsAsync because it uses Task.Wait and would deadlock.
+                    await call.ResponseStream.ToListAsync();
+                    Assert.Fail();
+                }
+                catch (RpcException e)
+                {
+                    Assert.AreEqual(StatusCode.Unknown, e.Status.StatusCode);
+                    Assert.AreEqual(echoStatus.Message, e.Status.Detail);
+                }
             }
 
             Console.WriteLine("Passed!");
diff --git a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
index 80dad9fdd9376a95af9efea1f733f9d8a81db06b..c9eca73452235b86e017aea8c8d898e5b0ff7072 100644
--- a/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
+++ b/src/csharp/Grpc.IntegrationTesting/WorkerServiceImpl.cs
@@ -64,7 +64,7 @@ namespace Grpc.Testing
             {
                 Stats = runner.GetStats(false),
                 Port = runner.BoundPort,
-                Cores = 0,  // TODO: set number of cores
+                Cores = Environment.ProcessorCount,
             });
                 
             while (await requestStream.MoveNext())
diff --git a/src/csharp/ext/grpc_csharp_ext.c b/src/csharp/ext/grpc_csharp_ext.c
index 5b8ff9b819b2f18387096ae268de6b82703402cf..4beef9ded8948df214a928395cca322192171636 100644
--- a/src/csharp/ext/grpc_csharp_ext.c
+++ b/src/csharp/ext/grpc_csharp_ext.c
@@ -806,11 +806,14 @@ GPR_EXPORT grpc_call_error GPR_CALLTYPE grpcsharp_call_set_credentials(
 /* Server */
 
 GPR_EXPORT grpc_server *GPR_CALLTYPE
-grpcsharp_server_create(grpc_completion_queue *cq,
-                        const grpc_channel_args *args) {
-  grpc_server *server = grpc_server_create(args, NULL);
+grpcsharp_server_create(const grpc_channel_args *args) {
+  return grpc_server_create(args, NULL);
+}
+
+GPR_EXPORT void GPR_CALLTYPE
+grpcsharp_server_register_completion_queue(grpc_server *server,
+                                           grpc_completion_queue *cq) {
   grpc_server_register_completion_queue(server, cq, NULL);
-  return server;
 }
 
 GPR_EXPORT int32_t GPR_CALLTYPE
diff --git a/src/node/ext/byte_buffer.cc b/src/node/ext/byte_buffer.cc
index 8e0b6916e9d5b564694dcc88934329518d56d2cd..3479a67702976377e58373bd8618d2aa7ac720fe 100644
--- a/src/node/ext/byte_buffer.cc
+++ b/src/node/ext/byte_buffer.cc
@@ -72,17 +72,13 @@ Local<Value> ByteBufferToBuffer(grpc_byte_buffer *buffer) {
   if (buffer == NULL) {
     return scope.Escape(Nan::Null());
   }
-  size_t length = grpc_byte_buffer_length(buffer);
-  char *result = new char[length];
-  size_t offset = 0;
   grpc_byte_buffer_reader reader;
   grpc_byte_buffer_reader_init(&reader, buffer);
-  gpr_slice next;
-  while (grpc_byte_buffer_reader_next(&reader, &next) != 0) {
-    memcpy(result + offset, GPR_SLICE_START_PTR(next), GPR_SLICE_LENGTH(next));
-    offset += GPR_SLICE_LENGTH(next);
-    gpr_slice_unref(next);
-  }
+  gpr_slice slice = grpc_byte_buffer_reader_readall(&reader);
+  size_t length = GPR_SLICE_LENGTH(slice);
+  char *result = new char[length];
+  memcpy(result, GPR_SLICE_START_PTR(slice), length);
+  gpr_slice_unref(slice);
   return scope.Escape(MakeFastBuffer(
       Nan::NewBuffer(result, length, delete_buffer, NULL).ToLocalChecked()));
 }
diff --git a/src/node/ext/server.cc b/src/node/ext/server.cc
index b9e1fe916013bd5f18a0d75475241df3f97ac0b7..dd1b777ac88b1bf9ca216817da525de4b51b586a 100644
--- a/src/node/ext/server.cc
+++ b/src/node/ext/server.cc
@@ -35,15 +35,15 @@
 
 #include "server.h"
 
-#include <node.h>
 #include <nan.h>
+#include <node.h>
 
 #include <vector>
+#include "call.h"
+#include "completion_queue_async_worker.h"
 #include "grpc/grpc.h"
 #include "grpc/grpc_security.h"
 #include "grpc/support/log.h"
-#include "call.h"
-#include "completion_queue_async_worker.h"
 #include "server_credentials.h"
 #include "timeval.h"
 
@@ -100,8 +100,8 @@ class NewCallOp : public Op {
     Nan::Set(obj, Nan::New("host").ToLocalChecked(),
              Nan::New(details.host).ToLocalChecked());
     Nan::Set(obj, Nan::New("deadline").ToLocalChecked(),
-             Nan::New<Date>(
-                 TimespecToMilliseconds(details.deadline)).ToLocalChecked());
+             Nan::New<Date>(TimespecToMilliseconds(details.deadline))
+                 .ToLocalChecked());
     Nan::Set(obj, Nan::New("metadata").ToLocalChecked(),
              ParseMetadata(&request_metadata));
     return scope.Escape(obj);
@@ -117,14 +117,13 @@ class NewCallOp : public Op {
   grpc_metadata_array request_metadata;
 
  protected:
-  std::string GetTypeString() const {
-    return "new_call";
-  }
+  std::string GetTypeString() const { return "new_call"; }
 };
 
 Server::Server(grpc_server *server) : wrapped_server(server) {
   shutdown_queue = grpc_completion_queue_create(NULL);
-  grpc_server_register_completion_queue(server, shutdown_queue, NULL);
+  grpc_server_register_non_listening_completion_queue(server, shutdown_queue,
+                                                      NULL);
 }
 
 Server::~Server() {
@@ -156,8 +155,7 @@ bool Server::HasInstance(Local<Value> val) {
 }
 
 void Server::ShutdownServer() {
-  grpc_server_shutdown_and_notify(this->wrapped_server,
-                                  this->shutdown_queue,
+  grpc_server_shutdown_and_notify(this->wrapped_server, this->shutdown_queue,
                                   NULL);
   grpc_server_cancel_all_calls(this->wrapped_server);
   grpc_completion_queue_pluck(this->shutdown_queue, NULL,
@@ -170,8 +168,8 @@ NAN_METHOD(Server::New) {
   if (!info.IsConstructCall()) {
     const int argc = 1;
     Local<Value> argv[argc] = {info[0]};
-    MaybeLocal<Object> maybe_instance = constructor->GetFunction()->NewInstance(
-        argc, argv);
+    MaybeLocal<Object> maybe_instance =
+        constructor->GetFunction()->NewInstance(argc, argv);
     if (maybe_instance.IsEmpty()) {
       // There's probably a pending exception
       return;
@@ -185,8 +183,9 @@ NAN_METHOD(Server::New) {
   grpc_channel_args *channel_args;
   if (!ParseChannelArgs(info[0], &channel_args)) {
     DeallocateChannelArgs(channel_args);
-    return Nan::ThrowTypeError("Server options must be an object with "
-                               "string keys and integer or string values");
+    return Nan::ThrowTypeError(
+        "Server options must be an object with "
+        "string keys and integer or string values");
   }
   wrapped_server = grpc_server_create(channel_args, NULL);
   DeallocateChannelArgs(channel_args);
@@ -218,8 +217,7 @@ NAN_METHOD(Server::RequestCall) {
 
 NAN_METHOD(Server::AddHttp2Port) {
   if (!HasInstance(info.This())) {
-    return Nan::ThrowTypeError(
-        "addHttp2Port can only be called on a Server");
+    return Nan::ThrowTypeError("addHttp2Port can only be called on a Server");
   }
   if (!info[0]->IsString()) {
     return Nan::ThrowTypeError(
@@ -239,8 +237,7 @@ NAN_METHOD(Server::AddHttp2Port) {
                                                *Utf8String(info[0]));
   } else {
     port = grpc_server_add_secure_http2_port(server->wrapped_server,
-                                             *Utf8String(info[0]),
-                                             creds);
+                                             *Utf8String(info[0]), creds);
   }
   info.GetReturnValue().Set(Nan::New<Number>(port));
 }
@@ -262,8 +259,7 @@ NAN_METHOD(Server::TryShutdown) {
   Server *server = ObjectWrap::Unwrap<Server>(info.This());
   unique_ptr<OpVec> ops(new OpVec());
   grpc_server_shutdown_and_notify(
-      server->wrapped_server,
-      CompletionQueueAsyncWorker::GetQueue(),
+      server->wrapped_server, CompletionQueueAsyncWorker::GetQueue(),
       new struct tag(new Nan::Callback(info[0].As<Function>()), ops.release(),
                      shared_ptr<Resources>(nullptr)));
   CompletionQueueAsyncWorker::Next();
diff --git a/src/objective-c/CronetFramework.podspec b/src/objective-c/CronetFramework.podspec
new file mode 100644
index 0000000000000000000000000000000000000000..20af7647f70c1d583fe9e7a51182a6cf37c57923
--- /dev/null
+++ b/src/objective-c/CronetFramework.podspec
@@ -0,0 +1,43 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
+Pod::Spec.new do |s|
+  s.name         = "CronetFramework"
+  s.version      = "0.0.2"
+  s.summary      = "Cronet, precompiled and used as a framework."
+  s.homepage     = "http://chromium.org"
+  s.license      = { :type => 'BSD' }
+  s.vendored_framework = "Cronet.framework"
+  s.author             = "The Chromium Authors"
+  s.ios.deployment_target = "8.0"
+  s.source       = { :http => 'https://storage.googleapis.com/grpc-precompiled-binaries/cronet/Cronet.framework.zip' }
+  s.preserve_paths = "Cronet.framework"
+  s.public_header_files = "Cronet.framework/Headers/**/*{.h}"
+end
diff --git a/src/objective-c/GRPCClient/GRPCCall+Cronet.h b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
new file mode 100644
index 0000000000000000000000000000000000000000..7f7fc6c2a96f0d395ae7cb48bc17fe4f5827371c
--- /dev/null
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.h
@@ -0,0 +1,55 @@
+/*
+ *
+ * Copyright 2016, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+#import <Cronet/Cronet.h>
+
+#import "GRPCCall.h"
+
+/**
+ * Methods for using cronet transport.
+ */
+@interface GRPCCall (Cronet)
+
+/**
+ * This method should be called before issuing the first RPC. It should be
+ * called only once. Create an instance of Cronet engine in your app elsewhere
+ * and pass the instance pointer in the cronet_engine parameter. Once set,
+ * all subsequent RPCs will use Cronet transport. The method is not thread
+ * safe.
+ */
++(void)useCronetWithEngine:(cronet_engine *)engine;
+
++(cronet_engine *)cronetEngine;
+
++(BOOL)isUsingCronet;
+
+@end
diff --git a/test/cpp/qps/async_unary_ping_pong_test.cc b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
similarity index 56%
rename from test/cpp/qps/async_unary_ping_pong_test.cc
rename to src/objective-c/GRPCClient/GRPCCall+Cronet.m
index 571a8b7300f7e08bdb5121042f38ed3d82ddddf7..69a410e95ac84909b1996e79cf903a8a9422893b 100644
--- a/test/cpp/qps/async_unary_ping_pong_test.cc
+++ b/src/objective-c/GRPCClient/GRPCCall+Cronet.m
@@ -1,6 +1,6 @@
 /*
  *
- * Copyright 2015, Google Inc.
+ * Copyright 2016, Google Inc.
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -31,47 +31,24 @@
  *
  */
 
-#include <set>
+#import "GRPCCall+Cronet.h"
 
-#include <grpc/support/log.h>
+static BOOL useCronet = NO;
+static cronet_engine *globalCronetEngine;
 
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
+@implementation GRPCCall (Cronet)
 
-namespace grpc {
-namespace testing {
-
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
-
-static void RunAsyncUnaryPingPong() {
-  gpr_log(GPR_INFO, "Running Async Unary Ping Pong");
-
-  ClientConfig client_config;
-  client_config.set_client_type(ASYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1);
-  client_config.set_client_channels(1);
-  client_config.set_async_client_threads(1);
-  client_config.set_rpc_type(UNARY);
-  client_config.mutable_load_params()->mutable_closed_loop();
-
-  ServerConfig server_config;
-  server_config.set_server_type(ASYNC_SERVER);
-  server_config.set_async_server_threads(1);
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPS(*result);
-  GetReporter()->ReportLatency(*result);
++ (void)useCronetWithEngine:(cronet_engine *)engine {
+  useCronet = YES;
+  globalCronetEngine = engine;
 }
-}  // namespace testing
-}  // namespace grpc
 
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
++ (cronet_engine *)cronetEngine {
+  return globalCronetEngine;
+}
 
-  grpc::testing::RunAsyncUnaryPingPong();
-  return 0;
++ (BOOL)isUsingCronet {
+  return useCronet;
 }
+
+@end
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.h b/src/objective-c/GRPCClient/private/GRPCChannel.h
index 70d1a9bd2ff63f2c63469bac04b0ed30cd2b3bcd..3219835d0293354278bdd1ec88dfe96c084915db 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.h
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.h
@@ -55,6 +55,12 @@ struct grpc_channel_credentials;
  */
 + (nullable GRPCChannel *)secureChannelWithHost:(nonnull NSString *)host;
 
+/**
+ * Creates a secure channel to the specified @c host using Cronet as a transport mechanism.
+ */
++ (nullable GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                          channelArgs:(NSDictionary *)channelArgs;
+
 /**
  * Creates a secure channel to the specified @c host using the specified @c credentials and
  * @c channelArgs. Only in tests should @c GRPC_SSL_TARGET_NAME_OVERRIDE_ARG channel arg be set.
diff --git a/src/objective-c/GRPCClient/private/GRPCChannel.m b/src/objective-c/GRPCClient/private/GRPCChannel.m
index 203ef58c0dd72cfff4d03575981858807d80a7df..e4e0dbe6d2c773f269f3614eb6608ec544d330ae 100644
--- a/src/objective-c/GRPCClient/private/GRPCChannel.m
+++ b/src/objective-c/GRPCClient/private/GRPCChannel.m
@@ -34,10 +34,13 @@
 #import "GRPCChannel.h"
 
 #include <grpc/grpc_security.h>
+#include <grpc/grpc_cronet.h>
 #include <grpc/support/alloc.h>
 #include <grpc/support/log.h>
 #include <grpc/support/string_util.h>
 
+#import <Cronet/Cronet.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 #import "GRPCCompletionQueue.h"
 
 void freeChannelArgs(grpc_channel_args *channel_args) {
@@ -99,6 +102,22 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   grpc_channel_args *_channelArgs;
 }
 
+- (instancetype)initWithHost:(NSString *)host
+                cronetEngine:(cronet_engine *)cronetEngine
+                 channelArgs:(NSDictionary *)channelArgs {
+  if (!host) {
+    [NSException raise:NSInvalidArgumentException format:@"host argument missing"];
+  }
+
+  if (self = [super init]) {
+    _channelArgs = buildChannelArgs(channelArgs);
+    _host = [host copy];
+    _unmanagedChannel = grpc_cronet_secure_channel_create(cronetEngine, _host.UTF8String, _channelArgs,
+                                                     NULL);
+  }
+
+  return self;
+}
 
 - (instancetype)initWithHost:(NSString *)host
                       secure:(BOOL)secure
@@ -133,6 +152,17 @@ grpc_channel_args * buildChannelArgs(NSDictionary *dictionary) {
   freeChannelArgs(_channelArgs);
 }
 
++ (GRPCChannel *)secureCronetChannelWithHost:(NSString *)host
+                                 channelArgs:(NSDictionary *)channelArgs {
+  cronet_engine *engine = [GRPCCall cronetEngine];
+  if (!engine) {
+    [NSException raise:NSInvalidArgumentException
+                format:@"cronet_engine is NULL. Set it first."];
+    return nil;
+  }
+  return [[GRPCChannel alloc] initWithHost:host cronetEngine:engine channelArgs:channelArgs];
+}
+
 + (GRPCChannel *)secureChannelWithHost:(NSString *)host {
   return [[GRPCChannel alloc] initWithHost:host secure:YES credentials:NULL channelArgs:NULL];
 }
diff --git a/src/objective-c/GRPCClient/private/GRPCHost.m b/src/objective-c/GRPCClient/private/GRPCHost.m
index 43166cbb5278bb3d4614d6596c4fda6cee6fff5e..7da508810ce0bc4b55a7cfdd23367b06da0f1f13 100644
--- a/src/objective-c/GRPCClient/private/GRPCHost.m
+++ b/src/objective-c/GRPCClient/private/GRPCHost.m
@@ -37,6 +37,7 @@
 #include <grpc/grpc_security.h>
 #import <GRPCClient/GRPCCall.h>
 #import <GRPCClient/GRPCCall+ChannelArg.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 
 #import "GRPCChannel.h"
 #import "GRPCCompletionQueue.h"
@@ -200,15 +201,21 @@ NS_ASSUME_NONNULL_BEGIN
 
 - (GRPCChannel *)newChannel {
   NSDictionary *args = [self channelArgs];
+  BOOL useCronet = [GRPCCall isUsingCronet];
   if (_secure) {
       GRPCChannel *channel;
       @synchronized(self) {
         if (_channelCreds == nil) {
           [self setTLSPEMRootCerts:nil withPrivateKey:nil withCertChain:nil error:nil];
         }
-        channel = [GRPCChannel secureChannelWithHost:_address
-                                          credentials:_channelCreds
-                                          channelArgs:args];
+        if (useCronet) {
+          channel = [GRPCChannel secureCronetChannelWithHost:_address
+                                                 channelArgs:args];
+        } else {
+          channel = [GRPCChannel secureChannelWithHost:_address
+                                            credentials:_channelCreds
+                                            channelArgs:args];
+        }
       }
       return channel;
   } else {
diff --git a/src/objective-c/tests/InteropTests.m b/src/objective-c/tests/InteropTests.m
index 4f096b9efabdc0301ac0edeb8d32cc26c7329fbf..781c500f73e976fcb8f479803829e13f8a823dd6 100644
--- a/src/objective-c/tests/InteropTests.m
+++ b/src/objective-c/tests/InteropTests.m
@@ -35,7 +35,9 @@
 
 #include <grpc/status.h>
 
+#import <Cronet/Cronet.h>
 #import <GRPCClient/GRPCCall+Tests.h>
+#import <GRPCClient/GRPCCall+Cronet.h>
 #import <ProtoRPC/ProtoRPC.h>
 #import <RemoteTest/Empty.pbobjc.h>
 #import <RemoteTest/Messages.pbobjc.h>
@@ -78,6 +80,8 @@
 
 #pragma mark Tests
 
+static cronet_engine *cronetEngine = NULL;
+
 @implementation InteropTests {
   RMTTestService *_service;
 }
@@ -88,6 +92,15 @@
 
 - (void)setUp {
   _service = self.class.host ? [RMTTestService serviceWithHost:self.class.host] : nil;
+#ifdef GRPC_COMPILE_WITH_CRONET
+  if (cronetEngine == NULL) {
+    // Cronet setup
+    [Cronet setHttp2Enabled:YES];
+    [Cronet start];
+    cronetEngine = [Cronet getGlobalEngine];
+    [GRPCCall useCronetWithEngine:cronetEngine];
+  }
+#endif
 }
 
 - (void)testEmptyUnaryRPC {
@@ -245,6 +258,8 @@
   [self waitForExpectationsWithTimeout:4 handler:nil];
 }
 
+#ifndef GRPC_COMPILE_WITH_CRONET
+// TODO(makdharma@): Fix this test
 - (void)testEmptyStreamRPC {
   XCTAssertNotNil(self.class.host);
   __weak XCTestExpectation *expectation = [self expectationWithDescription:@"EmptyStream"];
@@ -258,6 +273,7 @@
   }];
   [self waitForExpectationsWithTimeout:2 handler:nil];
 }
+#endif
 
 - (void)testCancelAfterBeginRPC {
   XCTAssertNotNil(self.class.host);
diff --git a/src/objective-c/tests/Podfile b/src/objective-c/tests/Podfile
index 7ec7a258982bff2fa1af5573ce51e309d918ff27..508641d681a4903c1fbb93754f440689b5a01ec9 100644
--- a/src/objective-c/tests/Podfile
+++ b/src/objective-c/tests/Podfile
@@ -3,6 +3,7 @@ platform :ios, '8.0'
 
 pod 'Protobuf', :path => "../../../third_party/protobuf"
 pod 'BoringSSL', :podspec => ".."
+pod 'CronetFramework', :podspec => ".."
 pod 'gRPC', :path => "../../.."
 pod 'RemoteTest', :path => "RemoteTestClient"
 
diff --git a/src/proto/census/census.options b/src/proto/census/census.options
new file mode 100644
index 0000000000000000000000000000000000000000..a1f80395c701c3ddcfa667d23fd6cfacc72b9f7d
--- /dev/null
+++ b/src/proto/census/census.options
@@ -0,0 +1,3 @@
+google.census.Tag.key max_size:255
+google.census.Tag.value max_size:255
+google.census.View.tag_key max_count:15
diff --git a/src/proto/census/census.proto b/src/proto/census/census.proto
new file mode 100644
index 0000000000000000000000000000000000000000..c869d851ff1394440a8c23ae156e1a27b6415d90
--- /dev/null
+++ b/src/proto/census/census.proto
@@ -0,0 +1,313 @@
+// Copyright 2016, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+syntax = "proto3";
+
+package google.census;
+
+// All the census protos.
+//
+// Nomenclature note: capitalized names below (like Metric) are protos.
+//
+// Census lets you define a Metric - something which can be measured, like the
+// latency of an RPC, the number of CPU cycles spent on an operation, or
+// anything else you care to measure. You can record individual instances of
+// measurements (a double value) for every metric of interest. These
+// individual measurements are aggregated together into an Aggregation. There
+// are two Aggregation types available: Distribution (describes the
+// distribution of all measurements, possibly with a histogram) and
+// IntervalStats (the count and mean of measurements across specified time
+// periods). An Aggregation is described by an AggregationDescriptor.
+//
+// You can define how your stats are broken down by Tag values and which
+// Aggregations to use through a View. The corresponding combination of
+// Metric/View/Aggregation which is available to census clients is called a
+// ViewAggregation.
+
+
+// The following two types are copied from
+// google/protobuf/{duration,timestamp}.proto. Ideally, we would be able to
+// import them, but this causes compilation issues on C-based systems
+// (e.g. https://koti.kapsi.fi/jpa/nanopb/), which cannot process the C++
+// headers generated from the standard protobuf distribution. See the relevant
+// proto files for full documentation of these types.
+
+message Duration {
+  // Signed seconds of the span of time. Must be from -315,576,000,000
+  // to +315,576,000,000 inclusive.
+  int64 seconds = 1;
+
+  // Signed fractions of a second at nanosecond resolution of the span
+  // of time. Durations less than one second are represented with a 0
+  // `seconds` field and a positive or negative `nanos` field. For durations
+  // of one second or more, a non-zero value for the `nanos` field must be
+  // of the same sign as the `seconds` field. Must be from -999,999,999
+  // to +999,999,999 inclusive.
+  int32 nanos = 2;
+}
+
+message Timestamp {
+  // Represents seconds of UTC time since Unix epoch
+  // 1970-01-01T00:00:00Z. Must be from from 0001-01-01T00:00:00Z to
+  // 9999-12-31T23:59:59Z inclusive.
+  int64 seconds = 1;
+
+  // Non-negative fractions of a second at nanosecond resolution. Negative
+  // second values with fractions must still have non-negative nanos values
+  // that count forward in time. Must be from 0 to 999,999,999
+  // inclusive.
+  int32 nanos = 2;
+}
+
+// Describes a metric
+message Metric {
+  // name of metric, e.g. rpc_latency, cpu.
+  string name = 1;
+
+  // More detailed description of the metric, used in documentation.
+  string description = 2;
+
+  // Fundamental units of measurement supported by Census
+  // TODO(aveitch): expand this to include other S.I. units?
+  message BasicUnit {
+    enum Measure {
+      UNKNOWN = 0;
+      BITS = 1;
+      BYTES = 2;
+      SECS = 3;
+      CORES = 4;
+      MAX_UNITS = 5;
+    }
+    Measure type = 1;
+  }
+
+  // MeasurementUnit lets you build compound units of the form
+  //   10^n * (A * B * ...) / (X * Y * ...),
+  // where the elements in the numerator and denominator are all BasicUnits.  A
+  // MeasurementUnit must have at least one BasicUnit in its numerator.
+  //
+  // To specify multiplication in the numerator or denominator, simply specify
+  // multiple numerator or denominator fields.  For example:
+  //
+  // - byte-seconds (i.e. bytes * seconds):
+  //     numerator: BYTES
+  //     numerator: SECS
+  //
+  // - events/sec^2 (i.e. rate of change of events/sec):
+  //     numerator: COUNT
+  //     denominator: SECS
+  //     denominator: SECS
+  //
+  // To specify multiples (in power of 10) units, specify a non-zero prefix
+  // value, for example:
+  //
+  // - MB/s (i.e. megabytes / s):
+  //     prefix: 6
+  //     numerator: BYTES
+  //     denominator: SECS
+  //
+  // - nanoseconds
+  //     prefix: -9
+  //     numerator: SECS
+  message MeasurementUnit {
+    int32 prefix = 1;
+    repeated BasicUnit numerator = 2;
+    repeated BasicUnit denominator = 3;
+  }
+
+  // The units in which the Metric value is reported.
+  MeasurementUnit unit = 3;
+
+  // Metrics will be assigned an ID when registered. Invalid if <= 0.
+  int32 id = 4;
+}
+
+// An Aggregation summarizes a series of individual Metric measurements, an
+// AggregationDescriptor describes an Aggregation.
+message AggregationDescriptor {
+  // At most one set of options. If neither option is set, a default type
+  // of Distribution (without a histogram component) will be used.
+  oneof options {
+    // Defines the histogram bucket boundaries for Distributions.
+    BucketBoundaries bucket_boundaries = 1;
+    // Defines the time windows to record for IntervalStats.
+    IntervalBoundaries interval_boundaries = 2;
+  }
+
+  // A Distribution may optionally contain a histogram of the values in the
+  // population. The bucket boundaries for that histogram is described by
+  // `bucket_boundaries`.
+  //
+  // Describes histogram bucket boundaries. Defines `size(bounds) + 1` (= N)
+  // buckets (for size(bounds) >= 1; if size(bounds) == 0, then no histogram
+  // will be defined. The boundaries for bucket index i are:
+  //
+  // [-infinity, bounds[i]) for i == 0
+  // [bounds[i-1], bounds[i]) for 0 < i < N-2
+  // [bounds[i-1], +infinity) for i == N-1
+  //
+  // i.e. an underflow bucket (number 0), zero or more finite buckets (1
+  // through N - 2, and an overflow bucket (N - 1), with inclusive lower
+  // bounds and exclusive upper bounds.
+  //
+  // There must be at least one element in `bounds`.  If `bounds` has only one
+  // element, there are no finite buckets, and that single element is the
+  // common boundary of the overflow and underflow buckets.
+  message BucketBoundaries {
+    // The values must be monotonically increasing.
+    repeated double bounds = 1;
+  }
+
+  // For Interval stats, describe the size of each window.
+  message IntervalBoundaries {
+    // For each time window, specify a duration in seconds.
+    repeated double window_size = 1;
+  }
+}
+
+// Distribution contains summary statistics for a population of values and,
+// optionally, a histogram representing the distribution of those values across
+// a specified set of histogram buckets, as defined in
+// Aggregation.bucket_options.
+//
+// The summary statistics are the count, mean, sum of the squared deviation from
+// the mean, the minimum, and the maximum of the set of population of values.
+//
+// Although it is not forbidden, it is generally a bad idea to include
+// non-finite values (infinities or NaNs) in the population of values, as this
+// will render the `mean` field meaningless.
+message Distribution {
+  // The number of values in the population. Must be non-negative.
+  int64 count = 1;
+
+  // The arithmetic mean of the values in the population. If `count` is zero
+  // then this field must be zero.
+  double mean = 2;
+
+  // Describes a range of population values.
+  message Range {
+    // The minimum of the population values.
+    double min = 1;
+    // The maximum of the population values.
+    double max = 2;
+  }
+
+  // The range of the population values. If `count` is zero, this field will not
+  // be defined.
+  Range range = 3;
+
+  // A Distribution may optionally contain a histogram of the values in the
+  // population.  The histogram is given in `bucket_count` as counts of values
+  // that fall into one of a sequence of non-overlapping buckets, as described
+  // by `AggregationDescriptor.options.bucket_boundaries`.
+  // The sum of the values in `bucket_counts` must equal the value in `count`.
+  //
+  // Bucket counts are given in order under the numbering scheme described
+  // above (the underflow bucket has number 0; the finite buckets, if any,
+  // have numbers 1 through N-2; the overflow bucket has number N-1).
+  //
+  // The size of `bucket_count` must be no greater than N as defined in
+  // `bucket_boundaries`.
+  //
+  // Any suffix of trailing zero bucket_count fields may be omitted.
+  repeated int64 bucket_count = 4;
+}
+
+// Record summary stats over various time windows.
+message IntervalStats {
+  // Summary statistic over a single time window.
+  message Window {
+    // The window duration.
+    Duration window_size = 1;
+    // The number of measurements in this window.
+    int64 count = 2;
+    // The arithmetic mean of all measurements in the window.
+    double mean = 3;
+  }
+
+  // Full set of windows for this metric.
+  repeated Window window = 1;
+}
+
+// A Tag: key-value pair.
+message Tag {
+  string key = 1;
+  string value = 2;
+}
+
+// A View specifies an Aggregation and a set of tag keys. The Aggregation will
+// be broken down by the unique set of matching tag values for each measurement.
+message View {
+  // Name of view.
+  string name = 1;
+
+  // More detailed description, for documentation purposes.
+  string description = 2;
+
+  // ID of Metric to associate with this View.
+  int32 metric_id = 3;
+
+  // Aggregation type to associate with this View.
+  AggregationDescriptor aggregation = 4;
+
+  // Tag keys to match with a given Metric. If no keys are specified, then all
+  // stats for the Metric are recorded. Keys must be unique.
+  repeated string tag_key = 5;
+}
+
+// An Aggregation summarizes a series of individual Metric measures.
+message Aggregation {
+  // Name of this aggregation.
+  string name = 1;
+
+  // More detailed description, for documentation purposes.
+  string description = 2;
+
+  // The data for this Aggregation.
+  oneof data {
+    Distribution distribution = 3;
+    IntervalStats interval_stats = 4;
+  }
+
+  // Tags associated with this Aggregation.
+  repeated Tag tag = 5;
+}
+
+// A ViewAggregations represents all the Aggregations for a particular view.
+message ViewAggregations {
+  // Aggregations - each will have a unique set of tag values for the tag_keys
+  // associated with the corresponding View.
+  repeated Aggregation aggregation = 1;
+
+  // Start and end timestamps over which the value was accumulated. These
+  // values are not relevant/defined for IntervalStats aggregations, which are
+  // always accumulated over a fixed time period.
+  Timestamp start = 2;
+  Timestamp end = 3;
+}
diff --git a/src/python/grpcio/grpc/__init__.py b/src/python/grpcio/grpc/__init__.py
index b844a14c48d7a7991d10c8c97a6c82012646b3dc..86447314b60fe3b3b6470853379bc8e479a863ce 100644
--- a/src/python/grpcio/grpc/__init__.py
+++ b/src/python/grpcio/grpc/__init__.py
@@ -27,5 +27,763 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
+"""gRPC's Python API."""
+
 __import__('pkg_resources').declare_namespace(__name__)
 
+import abc
+import enum
+
+import six
+
+from grpc._cython import cygrpc as _cygrpc
+
+
+############################## Future Interface  ###############################
+
+
+class FutureTimeoutError(Exception):
+  """Indicates that a method call on a Future timed out."""
+
+
+class FutureCancelledError(Exception):
+  """Indicates that the computation underlying a Future was cancelled."""
+
+
+class Future(six.with_metaclass(abc.ABCMeta)):
+  """A representation of a computation in another control flow.
+
+  Computations represented by a Future may be yet to be begun, may be ongoing,
+  or may have already completed.
+  """
+
+  @abc.abstractmethod
+  def cancel(self):
+    """Attempts to cancel the computation.
+
+    This method does not block.
+
+    Returns:
+      True if the computation has not yet begun, will not be allowed to take
+        place, and determination of both was possible without blocking. False
+        under all other circumstances including but not limited to the
+        computation's already having begun, the computation's already having
+        finished, and the computation's having been scheduled for execution on a
+        remote system for which a determination of whether or not it commenced
+        before being cancelled cannot be made without blocking.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def cancelled(self):
+    """Describes whether the computation was cancelled.
+
+    This method does not block.
+
+    Returns:
+      True if the computation was cancelled any time before its result became
+        immediately available. False under all other circumstances including but
+        not limited to this object's cancel method not having been called and
+        the computation's result having become immediately available.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def running(self):
+    """Describes whether the computation is taking place.
+
+    This method does not block.
+
+    Returns:
+      True if the computation is scheduled to take place in the future or is
+        taking place now, or False if the computation took place in the past or
+        was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def done(self):
+    """Describes whether the computation has taken place.
+
+    This method does not block.
+
+    Returns:
+      True if the computation is known to have either completed or have been
+        unscheduled or interrupted. False if the computation may possibly be
+        executing or scheduled to execute later.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def result(self, timeout=None):
+    """Accesses the outcome of the computation or raises its exception.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        finish or be cancelled, or None if this method should block until the
+        computation has finished or is cancelled no matter how long that takes.
+
+    Returns:
+      The return value of the computation.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+      Exception: If the computation raised an exception, this call will raise
+        the same exception.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def exception(self, timeout=None):
+    """Return the exception raised by the computation.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        terminate or be cancelled, or None if this method should block until
+        the computation is terminated or is cancelled no matter how long that
+        takes.
+
+    Returns:
+      The exception raised by the computation, or None if the computation did
+        not raise an exception.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def traceback(self, timeout=None):
+    """Access the traceback of the exception raised by the computation.
+
+    This method may return immediately or may block.
+
+    Args:
+      timeout: The length of time in seconds to wait for the computation to
+        terminate or be cancelled, or None if this method should block until
+        the computation is terminated or is cancelled no matter how long that
+        takes.
+
+    Returns:
+      The traceback of the exception raised by the computation, or None if the
+        computation did not raise an exception.
+
+    Raises:
+      FutureTimeoutError: If a timeout value is passed and the computation does
+        not terminate within the allotted time.
+      FutureCancelledError: If the computation was cancelled.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_done_callback(self, fn):
+    """Adds a function to be called at completion of the computation.
+
+    The callback will be passed this Future object describing the outcome of
+    the computation.
+
+    If the computation has already completed, the callback will be called
+    immediately.
+
+    Args:
+      fn: A callable taking this Future object as its single parameter.
+    """
+    raise NotImplementedError()
+
+
+################################  gRPC Enums  ##################################
+
+
+@enum.unique
+class ChannelConnectivity(enum.Enum):
+  """Mirrors grpc_connectivity_state in the gRPC Core.
+
+  Attributes:
+    IDLE: The channel is idle.
+    CONNECTING: The channel is connecting.
+    READY: The channel is ready to conduct RPCs.
+    TRANSIENT_FAILURE: The channel has seen a failure from which it expects to
+      recover.
+    FATAL_FAILURE: The channel has seen a failure from which it cannot recover.
+  """
+  IDLE              = (_cygrpc.ConnectivityState.idle, 'idle')
+  CONNECTING        = (_cygrpc.ConnectivityState.connecting, 'connecting')
+  READY             = (_cygrpc.ConnectivityState.ready, 'ready')
+  TRANSIENT_FAILURE = (
+      _cygrpc.ConnectivityState.transient_failure, 'transient failure')
+  FATAL_FAILURE     = (_cygrpc.ConnectivityState.fatal_failure, 'fatal failure')
+
+
+@enum.unique
+class StatusCode(enum.Enum):
+  """Mirrors grpc_status_code in the gRPC Core."""
+  OK                  = (_cygrpc.StatusCode.ok, 'ok')
+  CANCELLED           = (_cygrpc.StatusCode.cancelled, 'cancelled')
+  UNKNOWN             = (_cygrpc.StatusCode.unknown, 'unknown')
+  INVALID_ARGUMENT    = (
+      _cygrpc.StatusCode.invalid_argument, 'invalid argument')
+  DEADLINE_EXCEEDED   = (
+      _cygrpc.StatusCode.deadline_exceeded, 'deadline exceeded')
+  NOT_FOUND           = (_cygrpc.StatusCode.not_found, 'not found')
+  ALREADY_EXISTS      = (_cygrpc.StatusCode.already_exists, 'already exists')
+  PERMISSION_DENIED   = (
+      _cygrpc.StatusCode.permission_denied, 'permission denied')
+  RESOURCE_EXHAUSTED  = (
+      _cygrpc.StatusCode.resource_exhausted, 'resource exhausted')
+  FAILED_PRECONDITION = (
+      _cygrpc.StatusCode.failed_precondition, 'failed precondition')
+  ABORTED             = (_cygrpc.StatusCode.aborted, 'aborted')
+  OUT_OF_RANGE        = (_cygrpc.StatusCode.out_of_range, 'out of range')
+  UNIMPLEMENTED       = (_cygrpc.StatusCode.unimplemented, 'unimplemented')
+  INTERNAL            = (_cygrpc.StatusCode.internal, 'internal')
+  UNAVAILABLE         = (_cygrpc.StatusCode.unavailable, 'unavailable')
+  DATA_LOSS           = (_cygrpc.StatusCode.data_loss, 'data loss')
+  UNAUTHENTICATED     = (_cygrpc.StatusCode.unauthenticated, 'unauthenticated')
+
+
+#############################  gRPC Exceptions  ################################
+
+
+class RpcError(Exception):
+  """Raised by the gRPC library to indicate non-OK-status RPC termination."""
+
+
+##############################  Shared Context  ################################
+
+
+class RpcContext(six.with_metaclass(abc.ABCMeta)):
+  """Provides RPC-related information and action."""
+
+  @abc.abstractmethod
+  def is_active(self):
+    """Describes whether the RPC is active or has terminated."""
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def time_remaining(self):
+    """Describes the length of allowed time remaining for the RPC.
+
+    Returns:
+      A nonnegative float indicating the length of allowed time in seconds
+      remaining for the RPC to complete before it is considered to have timed
+      out, or None if no deadline was specified for the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def cancel(self):
+    """Cancels the RPC.
+
+    Idempotent and has no effect if the RPC has already terminated.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_callback(self, callback):
+    """Registers a callback to be called on RPC termination.
+
+    Args:
+      callback: A no-parameter callable to be called on RPC termination.
+
+    Returns:
+      True if the callback was added and will be called later; False if the
+        callback was not added and will not later be called (because the RPC
+        already terminated or some other reason).
+    """
+    raise NotImplementedError()
+
+
+#########################  Invocation-Side Context  ############################
+
+
+class Call(six.with_metaclass(abc.ABCMeta, RpcContext)):
+  """Invocation-side utility object for an RPC."""
+
+  @abc.abstractmethod
+  def initial_metadata(self):
+    """Accesses the initial metadata from the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The initial metadata as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def trailing_metadata(self):
+    """Accesses the trailing metadata from the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The trailing metadata as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def code(self):
+    """Accesses the status code emitted by the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The StatusCode value for the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def details(self):
+    """Accesses the details value emitted by the service-side of the RPC.
+
+    This method blocks until the value is available.
+
+    Returns:
+      The bytes of the details of the RPC.
+    """
+    raise NotImplementedError()
+
+
+########################  Multi-Callable Interfaces  ###########################
+
+
+class UnaryUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a unary-unary RPC."""
+
+  @abc.abstractmethod
+  def __call__(self, request, timeout=None, metadata=None, with_call=False):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      with_call: Whether or not to include return a Call for the RPC in addition
+        to the response.
+
+    Returns:
+      The response value for the RPC, and a Call for the RPC if with_call was
+        set to True at invocation.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def future(self, request, timeout=None, metadata=None):
+    """Asynchronously invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and a Future. In the event of
+        RPC completion, the return Future's result value will be the response
+        message of the RPC. Should the event terminate with non-OK status, the
+        returned Future's exception value will be an RpcError.
+    """
+    raise NotImplementedError()
+
+
+class UnaryStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a unary-stream RPC."""
+
+  @abc.abstractmethod
+  def __call__(self, request, timeout=None, metadata=None):
+    """Invokes the underlying RPC.
+
+    Args:
+      request: The request value for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and an iterator of response
+        values. Drawing response values from the returned iterator may raise
+        RpcError indicating termination of the RPC with non-OK status.
+    """
+    raise NotImplementedError()
+
+
+class StreamUnaryMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a stream-unary RPC in any call style."""
+
+  @abc.abstractmethod
+  def __call__(
+      self, request_iterator, timeout=None, metadata=None, with_call=False):
+    """Synchronously invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+      with_call: Whether or not to include return a Call for the RPC in addition
+        to the response.
+
+    Returns:
+      The response value for the RPC, and a Call for the RPC if with_call was
+        set to True at invocation.
+
+    Raises:
+      RpcError: Indicating that the RPC terminated with non-OK status. The
+        raised RpcError will also be a Call for the RPC affording the RPC's
+        metadata, status code, and details.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def future(self, request_iterator, timeout=None, metadata=None):
+    """Asynchronously invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and a Future. In the event of
+        RPC completion, the return Future's result value will be the response
+        message of the RPC. Should the event terminate with non-OK status, the
+        returned Future's exception value will be an RpcError.
+    """
+    raise NotImplementedError()
+
+
+class StreamStreamMultiCallable(six.with_metaclass(abc.ABCMeta)):
+  """Affords invoking a stream-stream RPC in any call style."""
+
+  @abc.abstractmethod
+  def __call__(self, request_iterator, timeout=None, metadata=None):
+    """Invokes the underlying RPC.
+
+    Args:
+      request_iterator: An iterator that yields request values for the RPC.
+      timeout: An optional duration of time in seconds to allow for the RPC.
+      metadata: An optional sequence of pairs of bytes to be transmitted to the
+        service-side of the RPC.
+
+    Returns:
+      An object that is both a Call for the RPC and an iterator of response
+        values. Drawing response values from the returned iterator may raise
+        RpcError indicating termination of the RPC with non-OK status.
+    """
+    raise NotImplementedError()
+
+
+#############################  Channel Interface  ##############################
+
+
+class Channel(six.with_metaclass(abc.ABCMeta)):
+  """Affords RPC invocation via generic methods."""
+
+  @abc.abstractmethod
+  def subscribe(self, callback, try_to_connect=False):
+    """Subscribes to this Channel's connectivity.
+
+    Args:
+      callback: A callable to be invoked and passed a ChannelConnectivity value
+        describing this Channel's connectivity. The callable will be invoked
+        immediately upon subscription and again for every change to this
+        Channel's connectivity thereafter until it is unsubscribed or this
+        Channel object goes out of scope.
+      try_to_connect: A boolean indicating whether or not this Channel should
+        attempt to connect if it is not already connected and ready to conduct
+        RPCs.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unsubscribe(self, callback):
+    """Unsubscribes a callback from this Channel's connectivity.
+
+    Args:
+      callback: A callable previously registered with this Channel from having
+        been passed to its "subscribe" method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unary_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a UnaryUnaryMultiCallable for a unary-unary method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A UnaryUnaryMultiCallable value for the named unary-unary method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def unary_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a UnaryStreamMultiCallable for a unary-stream method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A UnaryStreamMultiCallable value for the name unary-stream method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stream_unary(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a StreamUnaryMultiCallable for a stream-unary method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A StreamUnaryMultiCallable value for the named stream-unary method.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stream_stream(
+      self, method, request_serializer=None, response_deserializer=None):
+    """Creates a StreamStreamMultiCallable for a stream-stream method.
+
+    Args:
+      method: The name of the RPC method.
+
+    Returns:
+      A StreamStreamMultiCallable value for the named stream-stream method.
+    """
+    raise NotImplementedError()
+
+
+##########################  Service-Side Context  ##############################
+
+
+class ServicerContext(six.with_metaclass(abc.ABCMeta, RpcContext)):
+  """A context object passed to method implementations."""
+
+  @abc.abstractmethod
+  def invocation_metadata(self):
+    """Accesses the metadata from the invocation-side of the RPC.
+
+    Returns:
+      The invocation metadata object as a sequence of pairs of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def peer(self):
+    """Identifies the peer that invoked the RPC being serviced.
+
+    Returns:
+      A string identifying the peer that invoked the RPC being serviced.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def send_initial_metadata(self, initial_metadata):
+    """Sends the initial metadata value to the invocation-side of the RPC.
+
+    This method need not be called by method implementations if they have no
+    service-side initial metadata to transmit.
+
+    Args:
+      initial_metadata: The initial metadata of the RPC as a sequence of pairs
+        of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_trailing_metadata(self, trailing_metadata):
+    """Accepts the trailing metadata value of the RPC.
+
+    This method need not be called by method implementations if they have no
+    service-side trailing metadata to transmit.
+
+    Args:
+      trailing_metadata: The trailing metadata of the RPC as a sequence of pairs
+        of bytes.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_code(self, code):
+    """Accepts the status code of the RPC.
+
+    This method need not be called by method implementations if they wish the
+    gRPC runtime to determine the status code of the RPC.
+
+    Args:
+      code: The integer status code of the RPC to be transmitted to the
+        invocation side of the RPC.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def set_details(self, details):
+    """Accepts the service-side details of the RPC.
+
+    This method need not be called by method implementations if they have no
+    details to transmit.
+
+    Args:
+      details: The details bytes of the RPC to be transmitted to
+        the invocation side of the RPC.
+    """
+    raise NotImplementedError()
+
+
+#####################  Service-Side Handler Interfaces  ########################
+
+
+class RpcMethodHandler(six.with_metaclass(abc.ABCMeta)):
+  """An implementation of a single RPC method.
+
+  Attributes:
+    request_streaming: Whether the RPC supports exactly one request message or
+      any arbitrary number of request messages.
+    response_streaming: Whether the RPC supports exactly one response message or
+      any arbitrary number of response messages.
+    request_deserializer: A callable behavior that accepts a byte string and
+      returns an object suitable to be passed to this object's business logic,
+      or None to indicate that this object's business logic should be passed the
+      raw request bytes.
+    response_serializer: A callable behavior that accepts an object produced by
+      this object's business logic and returns a byte string, or None to
+      indicate that the byte strings produced by this object's business logic
+      should be transmitted on the wire as they are.
+    unary_unary: This object's application-specific business logic as a callable
+      value that takes a request value and a ServicerContext object and returns
+      a response value. Only non-None if both request_streaming and
+      response_streaming are False.
+    unary_stream: This object's application-specific business logic as a
+      callable value that takes a request value and a ServicerContext object and
+      returns an iterator of response values. Only non-None if request_streaming
+      is False and response_streaming is True.
+    stream_unary: This object's application-specific business logic as a
+      callable value that takes an iterator of request values and a
+      ServicerContext object and returns a response value. Only non-None if
+      request_streaming is True and response_streaming is False.
+    stream_stream: This object's application-specific business logic as a
+      callable value that takes an iterator of request values and a
+      ServicerContext object and returns an iterator of response values. Only
+      non-None if request_streaming and response_streaming are both True.
+  """
+
+
+class HandlerCallDetails(six.with_metaclass(abc.ABCMeta)):
+  """Describes an RPC that has just arrived for service.
+
+  Attributes:
+    method: The method name of the RPC.
+    invocation_metadata: The metadata from the invocation side of the RPC.
+  """
+
+
+class GenericRpcHandler(six.with_metaclass(abc.ABCMeta)):
+  """An implementation of arbitrarily many RPC methods."""
+
+  @abc.abstractmethod
+  def service(self, handler_call_details):
+    """Services an RPC (or not).
+
+    Args:
+      handler_call_details: A HandlerCallDetails describing the RPC.
+
+    Returns:
+      An RpcMethodHandler with which the RPC may be serviced, or None to
+        indicate that this object will not be servicing the RPC.
+    """
+    raise NotImplementedError()
+
+
+#############################  Server Interface  ###############################
+
+
+class Server(six.with_metaclass(abc.ABCMeta)):
+  """Services RPCs."""
+
+  @abc.abstractmethod
+  def add_generic_rpc_handlers(self, generic_rpc_handlers):
+    """Registers GenericRpcHandlers with this Server.
+
+    This method is only safe to call before the server is started.
+
+    Args:
+      generic_rpc_handlers: An iterable of GenericRpcHandlers that will be used
+        to service RPCs after this Server is started.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def add_insecure_port(self, address):
+    """Reserves a port for insecure RPC service once this Server becomes active.
+
+    This method may only be called before calling this Server's start method is
+    called.
+
+    Args:
+      address: The address for which to open a port.
+
+    Returns:
+      An integer port on which RPCs will be serviced after this link has been
+        started. This is typically the same number as the port number contained
+        in the passed address, but will likely be different if the port number
+        contained in the passed address was zero.
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def start(self):
+    """Starts this Server's service of RPCs.
+
+    This method may only be called while the server is not serving RPCs (i.e. it
+    is not idempotent).
+    """
+    raise NotImplementedError()
+
+  @abc.abstractmethod
+  def stop(self, grace):
+    """Stops this Server's service of RPCs.
+
+    All calls to this method immediately stop service of new RPCs. When existing
+    RPCs are aborted is controlled by the grace period parameter passed to this
+    method.
+
+    This method may be called at any time and is idempotent. Passing a smaller
+    grace value than has been passed in a previous call will have the effect of
+    stopping the Server sooner. Passing a larger grace value than has been
+    passed in a previous call will not have the effect of stopping the server
+    later.
+
+    Args:
+      grace: A duration of time in seconds to allow existing RPCs to complete
+        before being aborted by this Server's stopping. If None, this method
+        will block until the server is completely stopped.
+
+    Returns:
+      A threading.Event that will be set when this Server has completely
+      stopped. The returned event may not be set until after the full grace
+      period (if some ongoing RPC continues for the full length of the period)
+      of it may be set much sooner (such as if this Server had no RPCs underway
+      at the time it was stopped or if all RPCs that it had underway completed
+      very early in the grace period).
+    """
+    raise NotImplementedError()
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
index 66e6e6b549a3e6ffe9323ed7b7ec50f9ca00bb1f..d42c58050fbadd5cd1a3d1d651b4ec08898f33af 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc.pxi
@@ -336,6 +336,8 @@ cdef extern from "grpc/_cython/loader.h":
   void grpc_server_register_completion_queue(grpc_server *server,
                                              grpc_completion_queue *cq,
                                              void *reserved) nogil
+  void grpc_server_register_non_listening_completion_queue(
+      grpc_server *server, grpc_completion_queue *cq, void *reserved) nogil
   int grpc_server_add_insecure_http2_port(
       grpc_server *server, const char *addr) nogil
   void grpc_server_start(grpc_server *server) nogil
diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
index 8419a5906818a571512108a87a6b55c8d5f96523..55948755b2f86bf194f3ad3befa2b310ba42cc43 100644
--- a/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
+++ b/src/python/grpcio/grpc/_cython/_cygrpc/server.pyx.pxi
@@ -81,11 +81,20 @@ cdef class Server:
           self.c_server, queue.c_completion_queue, NULL)
     self.registered_completion_queues.append(queue)
 
+  def register_non_listening_completion_queue(
+      self, CompletionQueue queue not None):
+    if self.is_started:
+      raise ValueError("cannot register completion queues after start")
+    with nogil:
+      grpc_server_register_non_listening_completion_queue(
+          self.c_server, queue.c_completion_queue, NULL)
+    self.registered_completion_queues.append(queue)
+
   def start(self):
     if self.is_started:
       raise ValueError("the server has already started")
     self.backup_shutdown_queue = CompletionQueue()
-    self.register_completion_queue(self.backup_shutdown_queue)
+    self.register_non_listening_completion_queue(self.backup_shutdown_queue)
     self.is_started = True
     with nogil:
       grpc_server_start(self.c_server)
@@ -169,4 +178,3 @@ cdef class Server:
           time.sleep(0)
       with nogil:
         grpc_server_destroy(self.c_server)
-
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.c b/src/python/grpcio/grpc/_cython/imports.generated.c
index 09551472b5f4bb7fd65c0cc36726a8d8eff3af7b..f71cf128445477ddbe385ae331be1c84e3e2b420 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.c
+++ b/src/python/grpcio/grpc/_cython/imports.generated.c
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -386,6 +387,7 @@ void pygrpc_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
diff --git a/src/python/grpcio/grpc/_cython/imports.generated.h b/src/python/grpcio/grpc/_cython/imports.generated.h
index 6de295414ad6409aa254b4bb2d37a4e22861e061..a364075e9e3f2f77cf2cd8223814d27391b81d58 100644
--- a/src/python/grpcio/grpc/_cython/imports.generated.h
+++ b/src/python/grpcio/grpc/_cython/imports.generated.h
@@ -296,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
diff --git a/src/python/grpcio/grpc/framework/foundation/future.py b/src/python/grpcio/grpc/framework/foundation/future.py
index 92106161504941452f31978117052a5a2722460e..6fb58eadb60922c42476de27a848ad3a28ef9531 100644
--- a/src/python/grpcio/grpc/framework/foundation/future.py
+++ b/src/python/grpcio/grpc/framework/foundation/future.py
@@ -232,6 +232,6 @@ class Future(six.with_metaclass(abc.ABCMeta)):
     immediately.
 
     Args:
-      fn: A callable taking a this Future object as its single parameter.
+      fn: A callable taking this Future object as its single parameter.
     """
     raise NotImplementedError()
diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py
index f7a5ed8788406abd1e5113545a128fb2f7104fd1..d29d00b2eb649cb75c872643f34f924bea37e841 100644
--- a/src/python/grpcio/grpc_core_dependencies.py
+++ b/src/python/grpcio/grpc_core_dependencies.py
@@ -183,20 +183,28 @@ CORE_SOURCE_FILES = [
   'src/core/ext/transport/chttp2/transport/writing.c',
   'src/core/ext/transport/chttp2/alpn/alpn.c',
   'src/core/lib/http/httpcli_security_connector.c',
-  'src/core/lib/security/b64.c',
-  'src/core/lib/security/client_auth_filter.c',
-  'src/core/lib/security/credentials.c',
-  'src/core/lib/security/credentials_metadata.c',
-  'src/core/lib/security/credentials_posix.c',
-  'src/core/lib/security/credentials_win32.c',
-  'src/core/lib/security/google_default_credentials.c',
-  'src/core/lib/security/handshake.c',
-  'src/core/lib/security/json_token.c',
-  'src/core/lib/security/jwt_verifier.c',
-  'src/core/lib/security/secure_endpoint.c',
-  'src/core/lib/security/security_connector.c',
-  'src/core/lib/security/security_context.c',
-  'src/core/lib/security/server_auth_filter.c',
+  'src/core/lib/security/context/security_context.c',
+  'src/core/lib/security/credentials/composite/composite_credentials.c',
+  'src/core/lib/security/credentials/credentials.c',
+  'src/core/lib/security/credentials/credentials_metadata.c',
+  'src/core/lib/security/credentials/fake/fake_credentials.c',
+  'src/core/lib/security/credentials/google_default/credentials_posix.c',
+  'src/core/lib/security/credentials/google_default/credentials_win32.c',
+  'src/core/lib/security/credentials/google_default/google_default_credentials.c',
+  'src/core/lib/security/credentials/iam/iam_credentials.c',
+  'src/core/lib/security/credentials/jwt/json_token.c',
+  'src/core/lib/security/credentials/jwt/jwt_credentials.c',
+  'src/core/lib/security/credentials/jwt/jwt_verifier.c',
+  'src/core/lib/security/credentials/oauth2/oauth2_credentials.c',
+  'src/core/lib/security/credentials/plugin/plugin_credentials.c',
+  'src/core/lib/security/credentials/ssl/ssl_credentials.c',
+  'src/core/lib/security/transport/client_auth_filter.c',
+  'src/core/lib/security/transport/handshake.c',
+  'src/core/lib/security/transport/secure_endpoint.c',
+  'src/core/lib/security/transport/security_connector.c',
+  'src/core/lib/security/transport/server_auth_filter.c',
+  'src/core/lib/security/util/b64.c',
+  'src/core/lib/security/util/json_util.c',
   'src/core/lib/surface/init_secure.c',
   'src/core/lib/tsi/fake_transport_security.c',
   'src/core/lib/tsi/ssl_transport_security.c',
@@ -238,6 +246,7 @@ CORE_SOURCE_FILES = [
   'src/core/ext/load_reporting/load_reporting.c',
   'src/core/ext/load_reporting/load_reporting_filter.c',
   'src/core/ext/census/context.c',
+  'src/core/ext/census/gen/census.pb.c',
   'src/core/ext/census/grpc_context.c',
   'src/core/ext/census/grpc_filter.c',
   'src/core/ext/census/grpc_plugin.c',
diff --git a/src/python/grpcio/tests/tests.json b/src/python/grpcio/tests/tests.json
index 691062f25ac7725e1656a8bca39ea02099031991..1beb619f8795319ee6ceb300b39b0a04a0667710 100644
--- a/src/python/grpcio/tests/tests.json
+++ b/src/python/grpcio/tests/tests.json
@@ -5,6 +5,7 @@
   "_base_interface_test.SyncPeasyTest", 
   "_beta_features_test.BetaFeaturesTest", 
   "_beta_features_test.ContextManagementAndLifecycleTest", 
+  "_cancel_many_calls_test.CancelManyCallsTest",
   "_channel_test.ChannelTest", 
   "_connectivity_channel_test.ChannelConnectivityTest", 
   "_core_over_links_base_interface_test.AsyncEasyTest", 
diff --git a/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
new file mode 100644
index 0000000000000000000000000000000000000000..c1de7790145bf4b36324716ce2f5d7a94b9fa6fd
--- /dev/null
+++ b/src/python/grpcio/tests/unit/_cython/_cancel_many_calls_test.py
@@ -0,0 +1,222 @@
+# Copyright 2016, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""Test making many calls and immediately cancelling most of them."""
+
+import threading
+import unittest
+
+from grpc._cython import cygrpc
+from grpc.framework.foundation import logging_pool
+from tests.unit.framework.common import test_constants
+
+_INFINITE_FUTURE = cygrpc.Timespec(float('+inf'))
+_EMPTY_FLAGS = 0
+_EMPTY_METADATA = cygrpc.Metadata(())
+
+_SERVER_SHUTDOWN_TAG = 'server_shutdown'
+_REQUEST_CALL_TAG = 'request_call'
+_RECEIVE_CLOSE_ON_SERVER_TAG = 'receive_close_on_server'
+_RECEIVE_MESSAGE_TAG = 'receive_message'
+_SERVER_COMPLETE_CALL_TAG = 'server_complete_call'
+
+_SUCCESS_CALL_FRACTION = 1.0 / 8.0
+
+
+class _State(object):
+
+  def __init__(self):
+    self.condition = threading.Condition()
+    self.handlers_released = False
+    self.parked_handlers = 0
+    self.handled_rpcs = 0
+
+
+def _is_cancellation_event(event):
+  return (
+      event.tag is _RECEIVE_CLOSE_ON_SERVER_TAG and
+      event.batch_operations[0].received_cancelled)
+
+
+class _Handler(object):
+
+  def __init__(self, state, completion_queue, rpc_event):
+    self._state = state
+    self._lock = threading.Lock()
+    self._completion_queue = completion_queue
+    self._call = rpc_event.operation_call
+
+  def __call__(self):
+    with self._state.condition:
+      self._state.parked_handlers += 1
+      if self._state.parked_handlers == test_constants.THREAD_CONCURRENCY:
+        self._state.condition.notify_all()
+      while not self._state.handlers_released:
+        self._state.condition.wait()
+
+    with self._lock:
+      self._call.start_batch(
+          cygrpc.Operations(
+              (cygrpc.operation_receive_close_on_server(_EMPTY_FLAGS),)),
+          _RECEIVE_CLOSE_ON_SERVER_TAG)
+      self._call.start_batch(
+          cygrpc.Operations((cygrpc.operation_receive_message(_EMPTY_FLAGS),)),
+          _RECEIVE_MESSAGE_TAG)
+    first_event = self._completion_queue.poll()
+    if _is_cancellation_event(first_event):
+      self._completion_queue.poll()
+    else:
+      with self._lock:
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x79\x57', _EMPTY_FLAGS),
+            cygrpc.operation_send_status_from_server(
+                _EMPTY_METADATA, cygrpc.StatusCode.ok, b'test details!',
+                _EMPTY_FLAGS),
+        )
+        self._call.start_batch(
+            cygrpc.Operations(operations), _SERVER_COMPLETE_CALL_TAG)
+      self._completion_queue.poll()
+      self._completion_queue.poll()
+
+
+def _serve(state, server, server_completion_queue, thread_pool):
+  for _ in range(test_constants.RPC_CONCURRENCY):
+    call_completion_queue = cygrpc.CompletionQueue()
+    server.request_call(
+        call_completion_queue, server_completion_queue, _REQUEST_CALL_TAG)
+    rpc_event = server_completion_queue.poll()
+    thread_pool.submit(_Handler(state, call_completion_queue, rpc_event))
+    with state.condition:
+      state.handled_rpcs += 1
+      if test_constants.RPC_CONCURRENCY <= state.handled_rpcs:
+        state.condition.notify_all()
+  server_completion_queue.poll()
+
+
+class _QueueDriver(object):
+
+  def __init__(self, condition, completion_queue, due):
+    self._condition = condition
+    self._completion_queue = completion_queue
+    self._due = due
+    self._events = []
+    self._returned = False
+
+  def start(self):
+    def in_thread():
+      while True:
+        event = self._completion_queue.poll()
+        with self._condition:
+          self._events.append(event)
+          self._due.remove(event.tag)
+          self._condition.notify_all()
+          if not self._due:
+            self._returned = True
+            return
+    thread = threading.Thread(target=in_thread)
+    thread.start()
+
+  def events(self, at_least):
+    with self._condition:
+      while len(self._events) < at_least:
+        self._condition.wait()
+      return tuple(self._events)
+
+
+class CancelManyCallsTest(unittest.TestCase):
+
+  def testCancelManyCalls(self):
+    server_thread_pool = logging_pool.pool(test_constants.THREAD_CONCURRENCY)
+
+    server_completion_queue = cygrpc.CompletionQueue()
+    server = cygrpc.Server()
+    server.register_completion_queue(server_completion_queue)
+    port = server.add_http2_port('[::]:0')
+    server.start()
+    channel = cygrpc.Channel('localhost:{}'.format(port))
+
+    state = _State()
+
+    server_thread_args = (
+        state, server, server_completion_queue, server_thread_pool,)
+    server_thread = threading.Thread(target=_serve, args=server_thread_args)
+    server_thread.start()
+
+    client_condition = threading.Condition()
+    client_due = set()
+    client_completion_queue = cygrpc.CompletionQueue()
+    client_driver = _QueueDriver(
+        client_condition, client_completion_queue, client_due)
+    client_driver.start()
+
+    with client_condition:
+      client_calls = []
+      for index in range(test_constants.RPC_CONCURRENCY):
+        client_call = channel.create_call(
+            None, _EMPTY_FLAGS, client_completion_queue, b'/twinkies', None,
+            _INFINITE_FUTURE)
+        operations = (
+            cygrpc.operation_send_initial_metadata(
+                _EMPTY_METADATA, _EMPTY_FLAGS),
+            cygrpc.operation_send_message(b'\x45\x56', _EMPTY_FLAGS),
+            cygrpc.operation_send_close_from_client(_EMPTY_FLAGS),
+            cygrpc.operation_receive_initial_metadata(_EMPTY_FLAGS),
+            cygrpc.operation_receive_message(_EMPTY_FLAGS),
+            cygrpc.operation_receive_status_on_client(_EMPTY_FLAGS),
+        )
+        tag = 'client_complete_call_{0:04d}_tag'.format(index)
+        client_call.start_batch(cygrpc.Operations(operations), tag)
+        client_due.add(tag)
+        client_calls.append(client_call)
+
+    with state.condition:
+      while True:
+        if state.parked_handlers < test_constants.THREAD_CONCURRENCY:
+          state.condition.wait()
+        elif state.handled_rpcs < test_constants.RPC_CONCURRENCY:
+          state.condition.wait()
+        else:
+          state.handlers_released = True
+          state.condition.notify_all()
+          break
+
+    client_driver.events(
+        test_constants.RPC_CONCURRENCY * _SUCCESS_CALL_FRACTION)
+    with client_condition:
+      for client_call in client_calls:
+        client_call.cancel()
+
+    with state.condition:
+      server.shutdown(server_completion_queue, _SERVER_SHUTDOWN_TAG)
+
+
+if __name__ == '__main__':
+  unittest.main(verbosity=2)
diff --git a/src/python/grpcio/tests/unit/framework/common/test_control.py b/src/python/grpcio/tests/unit/framework/common/test_control.py
index ca5ba3a85423ea151c0f9ff03d981938de4c77ca..088e2f8b8850752a7def3decc73a02df1581faef 100644
--- a/src/python/grpcio/tests/unit/framework/common/test_control.py
+++ b/src/python/grpcio/tests/unit/framework/common/test_control.py
@@ -60,10 +60,16 @@ class Control(six.with_metaclass(abc.ABCMeta)):
 
 
 class PauseFailControl(Control):
-  """A Control that can be used to pause or fail code under control."""
+  """A Control that can be used to pause or fail code under control.
+
+  This object is only safe for use from two threads: one of the system under
+  test calling control and the other from the test system calling pause,
+  block_until_paused, and fail.
+  """
 
   def __init__(self):
     self._condition = threading.Condition()
+    self._pause = False
     self._paused = False
     self._fail = False
 
@@ -72,19 +78,31 @@ class PauseFailControl(Control):
       if self._fail:
         raise Defect()
 
-      while self._paused:
+      while self._pause:
+        self._paused = True
+        self._condition.notify_all()
         self._condition.wait()
+      self._paused = False
 
   @contextlib.contextmanager
   def pause(self):
     """Pauses code under control while controlling code is in context."""
     with self._condition:
-      self._paused = True
+      self._pause = True
     yield
     with self._condition:
-      self._paused = False
+      self._pause = False
       self._condition.notify_all()
 
+  def block_until_paused(self):
+    """Blocks controlling code until code under control is paused.
+
+    May only be called within the context of a pause call.
+    """
+    with self._condition:
+      while not self._paused:
+        self._condition.wait()
+
   @contextlib.contextmanager
   def fail(self):
     """Fails code under control while controlling code is in context."""
diff --git a/src/ruby/ext/grpc/rb_call.c b/src/ruby/ext/grpc/rb_call.c
index 1b06273af9d41c5769b30b6ace297ae0c1d8f9a7..b436057c16353ac90551eec2bd3283b029117fd0 100644
--- a/src/ruby/ext/grpc/rb_call.c
+++ b/src/ruby/ext/grpc/rb_call.c
@@ -101,30 +101,14 @@ static VALUE sym_message;
 static VALUE sym_status;
 static VALUE sym_cancelled;
 
-/* hash_all_calls is a hash of Call address -> reference count that is used to
- * track the creation and destruction of rb_call instances.
- */
-static VALUE hash_all_calls;
-
 /* Destroys a Call. */
 static void grpc_rb_call_destroy(void *p) {
-  grpc_call *call = NULL;
-  VALUE ref_count = Qnil;
+  grpc_call* call = NULL;
   if (p == NULL) {
     return;
-  };
-  call = (grpc_call *)p;
-
-  ref_count = rb_hash_aref(hash_all_calls, OFFT2NUM((VALUE)call));
-  if (ref_count == Qnil) {
-    return; /* No longer in the hash, so already deleted */
-  } else if (NUM2UINT(ref_count) == 1) {
-    rb_hash_delete(hash_all_calls, OFFT2NUM((VALUE)call));
-    grpc_call_destroy(call);
-  } else {
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)call),
-                 UINT2NUM(NUM2UINT(ref_count) - 1));
   }
+  call = (grpc_call *)p;
+  grpc_call_destroy(call);
 }
 
 static size_t md_ary_datasize(const void *p) {
@@ -151,7 +135,7 @@ static const rb_data_type_t grpc_rb_md_ary_data_type = {
      * touches a hash object.
      * TODO(yugui) Directly use st_table and call the free function earlier?
      */
-    0,
+     0,
 #endif
 };
 
@@ -163,12 +147,7 @@ static const rb_data_type_t grpc_call_data_type = {
     NULL,
     NULL,
 #ifdef RUBY_TYPED_FREE_IMMEDIATELY
-    /* it is unsafe to specify RUBY_TYPED_FREE_IMMEDIATELY because
-     * grpc_rb_call_destroy
-     * touches a hash object.
-     * TODO(yugui) Directly use st_table and call the free function earlier?
-     */
-    0,
+    RUBY_TYPED_FREE_IMMEDIATELY
 #endif
 };
 
@@ -190,6 +169,11 @@ const char *grpc_call_error_detail_of(grpc_call_error err) {
 static VALUE grpc_rb_call_cancel(VALUE self) {
   grpc_call *call = NULL;
   grpc_call_error err;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    //This call has been closed
+    return Qnil;
+  }
+
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   err = grpc_call_cancel(call, NULL);
   if (err != GRPC_CALL_OK) {
@@ -200,11 +184,29 @@ static VALUE grpc_rb_call_cancel(VALUE self) {
   return Qnil;
 }
 
+/* Releases the c-level resources associated with a call
+   Once a call has been closed, no further requests can be
+   processed.
+*/
+static VALUE grpc_rb_call_close(VALUE self) {
+  grpc_call *call = NULL;
+  TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
+  if(call != NULL) {
+    grpc_call_destroy(call);
+    RTYPEDDATA_DATA(self) = NULL;
+  }
+  return Qnil;
+}
+
 /* Called to obtain the peer that this call is connected to. */
 static VALUE grpc_rb_call_get_peer(VALUE self) {
   VALUE res = Qnil;
   grpc_call *call = NULL;
   char *peer = NULL;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot get peer value on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   peer = grpc_call_get_peer(call);
   res = rb_str_new2(peer);
@@ -218,6 +220,10 @@ static VALUE grpc_rb_call_get_peer_cert(VALUE self) {
   grpc_call *call = NULL;
   VALUE res = Qnil;
   grpc_auth_context *ctx = NULL;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot get peer cert on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
 
   ctx = grpc_call_auth_context(call);
@@ -323,6 +329,10 @@ static VALUE grpc_rb_call_set_credentials(VALUE self, VALUE credentials) {
   grpc_call *call = NULL;
   grpc_call_credentials *creds;
   grpc_call_error err;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot set credentials of closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
   creds = grpc_rb_get_wrapped_call_credentials(credentials);
   err = grpc_call_set_credentials(call, creds);
@@ -731,7 +741,7 @@ static VALUE grpc_run_batch_stack_build_result(run_batch_stack *st) {
    }
    tag = Object.new
    timeout = 10
-   call.start_batch(cqueue, tag, timeout, ops)
+   call.start_batch(cq, tag, timeout, ops)
 
    Start a batch of operations defined in the array ops; when complete, post a
    completion of type 'tag' to the completion queue bound to the call.
@@ -749,6 +759,10 @@ static VALUE grpc_rb_call_run_batch(VALUE self, VALUE cqueue, VALUE tag,
   VALUE result = Qnil;
   VALUE rb_write_flag = rb_ivar_get(self, id_write_flag);
   unsigned write_flag = 0;
+  if (RTYPEDDATA_DATA(self) == NULL) {
+    rb_raise(grpc_rb_eCallError, "Cannot run batch on closed call");
+    return Qnil;
+  }
   TypedData_Get_Struct(self, grpc_call, &grpc_call_data_type, call);
 
   /* Validate the ops args, adding them to a ruby array */
@@ -888,6 +902,7 @@ void Init_grpc_call() {
   /* Add ruby analogues of the Call methods. */
   rb_define_method(grpc_rb_cCall, "run_batch", grpc_rb_call_run_batch, 4);
   rb_define_method(grpc_rb_cCall, "cancel", grpc_rb_call_cancel, 0);
+  rb_define_method(grpc_rb_cCall, "close", grpc_rb_call_close, 0);
   rb_define_method(grpc_rb_cCall, "peer", grpc_rb_call_get_peer, 0);
   rb_define_method(grpc_rb_cCall, "peer_cert", grpc_rb_call_get_peer_cert, 0);
   rb_define_method(grpc_rb_cCall, "status", grpc_rb_call_get_status, 0);
@@ -925,11 +940,6 @@ void Init_grpc_call() {
       "BatchResult", "send_message", "send_metadata", "send_close",
       "send_status", "message", "metadata", "status", "cancelled", NULL);
 
-  /* The hash for reference counting calls, to ensure they can't be destroyed
-   * more than once */
-  hash_all_calls = rb_hash_new();
-  rb_define_const(grpc_rb_cCall, "INTERNAL_ALL_CALLs", hash_all_calls);
-
   Init_grpc_error_codes();
   Init_grpc_op_codes();
   Init_grpc_write_flags();
@@ -944,16 +954,8 @@ grpc_call *grpc_rb_get_wrapped_call(VALUE v) {
 
 /* Obtains the wrapped object for a given call */
 VALUE grpc_rb_wrap_call(grpc_call *c) {
-  VALUE obj = Qnil;
   if (c == NULL) {
     return Qnil;
   }
-  obj = rb_hash_aref(hash_all_calls, OFFT2NUM((VALUE)c));
-  if (obj == Qnil) { /* Not in the hash add it */
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)c), UINT2NUM(1));
-  } else {
-    rb_hash_aset(hash_all_calls, OFFT2NUM((VALUE)c),
-                 UINT2NUM(NUM2UINT(obj) + 1));
-  }
   return TypedData_Wrap_Struct(grpc_rb_cCall, &grpc_call_data_type, c);
 }
diff --git a/src/ruby/ext/grpc/rb_completion_queue.c b/src/ruby/ext/grpc/rb_completion_queue.c
index b6ddbe88dc895bc9a25dc1399f537d30e1de4f96..9466402db0c6b4887698b704e8b7a60347a23f79 100644
--- a/src/ruby/ext/grpc/rb_completion_queue.c
+++ b/src/ruby/ext/grpc/rb_completion_queue.c
@@ -150,6 +150,14 @@ static rb_data_type_t grpc_rb_completion_queue_data_type = {
 #endif
 };
 
+/* Releases the c-level resources associated with a completion queue */
+static VALUE grpc_rb_completion_queue_close(VALUE self) {
+  grpc_completion_queue* cq = grpc_rb_get_wrapped_completion_queue(self);
+  grpc_rb_completion_queue_destroy(cq);
+  RTYPEDDATA_DATA(self) = NULL;
+  return Qnil;
+}
+
 /* Allocates a completion queue. */
 static VALUE grpc_rb_completion_queue_alloc(VALUE cls) {
   grpc_completion_queue *cq = grpc_completion_queue_create(NULL);
@@ -212,6 +220,11 @@ void Init_grpc_completion_queue() {
      this func, so no separate initialization step is necessary. */
   rb_define_alloc_func(grpc_rb_cCompletionQueue,
                        grpc_rb_completion_queue_alloc);
+
+  /* close: Provides a way to close the underlying file descriptor without
+     waiting for ruby garbage collection. */
+  rb_define_method(grpc_rb_cCompletionQueue, "close",
+                   grpc_rb_completion_queue_close, 0);
 }
 
 /* Gets the wrapped completion queue from the ruby wrapper */
diff --git a/src/ruby/ext/grpc/rb_grpc.c b/src/ruby/ext/grpc/rb_grpc.c
index 06a07ac646336796eb238ca10de06d375b3a855a..9246893f9fb5aaff0d21805f9d3d10a825e16c33 100644
--- a/src/ruby/ext/grpc/rb_grpc.c
+++ b/src/ruby/ext/grpc/rb_grpc.c
@@ -318,7 +318,7 @@ void Init_grpc_c() {
   grpc_rb_mGrpcCore = rb_define_module_under(grpc_rb_mGRPC, "Core");
   grpc_rb_sNewServerRpc =
       rb_struct_define("NewServerRpc", "method", "host",
-                       "deadline", "metadata", "call", NULL);
+                       "deadline", "metadata", "call", "cq", NULL);
   grpc_rb_sStatus =
       rb_struct_define("Status", "code", "details", "metadata", NULL);
   sym_code = ID2SYM(rb_intern("code"));
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
index cebbe8c40feba774187743616512b427f310a4b8..3b62984defad4dd03aec88436a8e1cdc45f32a7d 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c
@@ -115,6 +115,7 @@ grpc_server_register_method_type grpc_server_register_method_import;
 grpc_server_request_registered_call_type grpc_server_request_registered_call_import;
 grpc_server_create_type grpc_server_create_import;
 grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
+grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
 grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 grpc_server_start_type grpc_server_start_import;
 grpc_server_shutdown_and_notify_type grpc_server_shutdown_and_notify_import;
@@ -382,6 +383,7 @@ void grpc_rb_load_imports(HMODULE library) {
   grpc_server_request_registered_call_import = (grpc_server_request_registered_call_type) GetProcAddress(library, "grpc_server_request_registered_call");
   grpc_server_create_import = (grpc_server_create_type) GetProcAddress(library, "grpc_server_create");
   grpc_server_register_completion_queue_import = (grpc_server_register_completion_queue_type) GetProcAddress(library, "grpc_server_register_completion_queue");
+  grpc_server_register_non_listening_completion_queue_import = (grpc_server_register_non_listening_completion_queue_type) GetProcAddress(library, "grpc_server_register_non_listening_completion_queue");
   grpc_server_add_insecure_http2_port_import = (grpc_server_add_insecure_http2_port_type) GetProcAddress(library, "grpc_server_add_insecure_http2_port");
   grpc_server_start_import = (grpc_server_start_type) GetProcAddress(library, "grpc_server_start");
   grpc_server_shutdown_and_notify_import = (grpc_server_shutdown_and_notify_type) GetProcAddress(library, "grpc_server_shutdown_and_notify");
diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
index d7ea6c574c8b8952725a7b2450677049373da521..1428e6d71c347576327a7607b89063d19a06c9c5 100644
--- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h
+++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h
@@ -296,6 +296,9 @@ extern grpc_server_create_type grpc_server_create_import;
 typedef void(*grpc_server_register_completion_queue_type)(grpc_server *server, grpc_completion_queue *cq, void *reserved);
 extern grpc_server_register_completion_queue_type grpc_server_register_completion_queue_import;
 #define grpc_server_register_completion_queue grpc_server_register_completion_queue_import
+typedef void(*grpc_server_register_non_listening_completion_queue_type)(grpc_server *server, grpc_completion_queue *q, void *reserved);
+extern grpc_server_register_non_listening_completion_queue_type grpc_server_register_non_listening_completion_queue_import;
+#define grpc_server_register_non_listening_completion_queue grpc_server_register_non_listening_completion_queue_import
 typedef int(*grpc_server_add_insecure_http2_port_type)(grpc_server *server, const char *addr);
 extern grpc_server_add_insecure_http2_port_type grpc_server_add_insecure_http2_port_import;
 #define grpc_server_add_insecure_http2_port grpc_server_add_insecure_http2_port_import
diff --git a/src/ruby/ext/grpc/rb_server.c b/src/ruby/ext/grpc/rb_server.c
index 0899feb685db6f0c718351f5f49c76b7b15c9ee2..f108b8acfcdbc1160d7eafb43a3fddb1b50de1f0 100644
--- a/src/ruby/ext/grpc/rb_server.c
+++ b/src/ruby/ext/grpc/rb_server.c
@@ -234,7 +234,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
     err = grpc_server_request_call(
         s->wrapped, &call, &st.details, &st.md_ary,
         grpc_rb_get_wrapped_completion_queue(cqueue),
-        grpc_rb_get_wrapped_completion_queue(cqueue),
+        grpc_rb_get_wrapped_completion_queue(s->mark),
         ROBJECT(tag_new));
     if (err != GRPC_CALL_OK) {
       grpc_request_call_stack_cleanup(&st);
@@ -244,7 +244,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
       return Qnil;
     }
 
-    ev = grpc_rb_completion_queue_pluck_event(cqueue, tag_new, timeout);
+    ev = grpc_rb_completion_queue_pluck_event(s->mark, tag_new, timeout);
     if (ev.type == GRPC_QUEUE_TIMEOUT) {
       grpc_request_call_stack_cleanup(&st);
       return Qnil;
@@ -262,7 +262,7 @@ static VALUE grpc_rb_server_request_call(VALUE self, VALUE cqueue,
         rb_str_new2(st.details.host),
         rb_funcall(rb_cTime, id_at, 2, INT2NUM(deadline.tv_sec),
                    INT2NUM(deadline.tv_nsec)),
-        grpc_rb_md_ary_to_h(&st.md_ary), grpc_rb_wrap_call(call), NULL);
+        grpc_rb_md_ary_to_h(&st.md_ary), grpc_rb_wrap_call(call), cqueue, NULL);
     grpc_request_call_stack_cleanup(&st);
     return result;
   }
diff --git a/src/ruby/lib/grpc/errors.rb b/src/ruby/lib/grpc/errors.rb
index a1dd1e3e9d8d18d8c87945b7fbbece5a21d73d8e..23b2bb7e127f707a645acaabfff19f3e9de24886 100644
--- a/src/ruby/lib/grpc/errors.rb
+++ b/src/ruby/lib/grpc/errors.rb
@@ -40,11 +40,12 @@ module GRPC
 
     # @param code [Numeric] the status code
     # @param details [String] the details of the exception
-    def initialize(code, details = 'unknown cause', **kw)
+    # @param metadata [Hash] the error's metadata
+    def initialize(code, details = 'unknown cause', metadata = {})
       super("#{code}:#{details}")
       @code = code
       @details = details
-      @metadata = kw
+      @metadata = metadata
     end
 
     # Converts the exception to a GRPC::Status for use in the networking
diff --git a/src/ruby/lib/grpc/generic/active_call.rb b/src/ruby/lib/grpc/generic/active_call.rb
index 7fe588bd4c8af679c72e8dd3828d3054497c6427..b03ddbc193c1ed904424b7689a07391bb42e8719 100644
--- a/src/ruby/lib/grpc/generic/active_call.rb
+++ b/src/ruby/lib/grpc/generic/active_call.rb
@@ -44,7 +44,7 @@ class Struct
         # raise BadStatus, propagating the metadata if present.
         md = status.metadata
         with_sym_keys = Hash[md.each_pair.collect { |x, y| [x.to_sym, y] }]
-        fail GRPC::BadStatus.new(status.code, status.details, **with_sym_keys)
+        fail GRPC::BadStatus.new(status.code, status.details, with_sym_keys)
       end
       status
     end
@@ -76,14 +76,15 @@ module GRPC
     #
     # @param call [Call] a call on which to start and invocation
     # @param q [CompletionQueue] the completion queue
-    def self.client_invoke(call, q, **kw)
+    # @param metadata [Hash] the metadata
+    def self.client_invoke(call, q, metadata = {})
       fail(TypeError, '!Core::Call') unless call.is_a? Core::Call
       unless q.is_a? Core::CompletionQueue
         fail(TypeError, '!Core::CompletionQueue')
       end
       metadata_tag = Object.new
       call.run_batch(q, metadata_tag, INFINITE_FUTURE,
-                     SEND_INITIAL_METADATA => kw)
+                     SEND_INITIAL_METADATA => metadata)
       metadata_tag
     end
 
@@ -102,7 +103,7 @@ module GRPC
     #
     # @param call [Call] the call used by the ActiveCall
     # @param q [CompletionQueue] the completion queue used to accept
-    #          the call
+    #          the call.  This queue will be closed on call completion.
     # @param marshal [Function] f(obj)->string that marshal requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
     # @param deadline [Fixnum] the deadline for the call to complete
@@ -190,6 +191,8 @@ module GRPC
       @call.status = batch_result.status
       op_is_done
       batch_result.check_status
+      @call.close
+      @cq.close
     end
 
     # remote_send sends a request to the remote endpoint.
@@ -211,13 +214,12 @@ module GRPC
     # @param details [String] details
     # @param assert_finished [true, false] when true(default), waits for
     # FINISHED.
-    #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    def send_status(code = OK, details = '', assert_finished = false, **kw)
+    # @param metadata [Hash] metadata to send to the server. If a value is a
+    # list, mulitple metadata for its key are sent
+    def send_status(code = OK, details = '', assert_finished = false,
+                    metadata: {})
       ops = {
-        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, kw)
+        SEND_STATUS_FROM_SERVER => Struct::Status.new(code, details, metadata)
       }
       ops[RECV_CLOSE_ON_SERVER] = nil if assert_finished
       @call.run_batch(@cq, self, INFINITE_FUTURE, ops)
@@ -311,14 +313,12 @@ module GRPC
     # request_response sends a request to a GRPC server, and returns the
     # response.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def request_response(req, **kw)
-      start_call(**kw) unless @started
+    def request_response(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       response = remote_read
@@ -337,14 +337,12 @@ module GRPC
     # array of marshallable objects; in typical case it will be an Enumerable
     # that allows dynamic construction of the marshallable objects.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Object] the response received from the server
-    def client_streamer(requests, **kw)
-      start_call(**kw) unless @started
+    def client_streamer(requests, metadata: {})
+      start_call(metadata) unless @started
       requests.each { |r| remote_send(r) }
       writes_done(false)
       response = remote_read
@@ -365,15 +363,12 @@ module GRPC
     # it is executed with each response as the argument and no result is
     # returned.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    # any keyword arguments are treated as metadata to be sent to the server.
-    #
     # @param req [Object] the request sent to the server
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator|nil] a response Enumerator
-    def server_streamer(req, **kw)
-      start_call(**kw) unless @started
+    def server_streamer(req, metadata: {})
+      start_call(metadata) unless @started
       remote_send(req)
       writes_done(false)
       replies = enum_for(:each_remote_read_then_finish)
@@ -407,14 +402,12 @@ module GRPC
     # the_call#writes_done has been called, otherwise the block will loop
     # forever.
     #
-    # == Keyword Arguments ==
-    # any keyword arguments are treated as metadata to be sent to the server
-    # if a keyword value is a list, multiple metadata for it's key are sent
-    #
     # @param requests [Object] an Enumerable of requests to send
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
     # @return [Enumerator, nil] a response Enumerator
-    def bidi_streamer(requests, **kw, &blk)
-      start_call(**kw) unless @started
+    def bidi_streamer(requests, metadata: {}, &blk)
+      start_call(metadata) unless @started
       bd = BidiCall.new(@call, @cq, @marshal, @unmarshal,
                         metadata_tag: @metadata_tag)
       @metadata_tag = nil  # run_on_client ensures metadata is read
@@ -453,9 +446,11 @@ module GRPC
     private
 
     # Starts the call if not already started
-    def start_call(**kw)
+    # @param metadata [Hash] metadata to be sent to the server. If a value is
+    # a list, multiple metadata for its key are sent
+    def start_call(metadata = {})
       return if @started
-      @metadata_tag = ActiveCall.client_invoke(@call, @cq, **kw)
+      @metadata_tag = ActiveCall.client_invoke(@call, @cq, metadata)
       @started = true
     end
 
diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb
index 1f6d5f365d75c98011f4ef4c09ebc718431d78aa..238f409a1d6019a16f5f49d71f6bc2904fa38a4f 100644
--- a/src/ruby/lib/grpc/generic/bidi_call.rb
+++ b/src/ruby/lib/grpc/generic/bidi_call.rb
@@ -69,6 +69,10 @@ module GRPC
       @readq = Queue.new
       @unmarshal = unmarshal
       @metadata_tag = metadata_tag
+      @reads_complete = false
+      @writes_complete = false
+      @complete = false
+      @done_mutex = Mutex.new
     end
 
     # Begins orchestration of the Bidi stream for a client sending requests.
@@ -115,6 +119,16 @@ module GRPC
       @op_notifier.notify(self)
     end
 
+    # signals that a bidi operation is complete (read + write)
+    def finished
+      @done_mutex.synchronize do
+        return unless @reads_complete && @writes_complete && !@complete
+        @call.close
+        @cq.close
+        @complete = true
+      end
+    end
+
     # performs a read using @call.run_batch, ensures metadata is set up
     def read_using_run_batch
       ops = { RECV_MESSAGE => nil }
@@ -163,12 +177,16 @@ module GRPC
                         SEND_CLOSE_FROM_CLIENT => nil)
         GRPC.logger.debug('bidi-write-loop: done')
         notify_done
+        @writes_complete = true
+        finished
       end
       GRPC.logger.debug('bidi-write-loop: finished')
     rescue StandardError => e
       GRPC.logger.warn('bidi-write-loop: failed')
       GRPC.logger.warn(e)
       notify_done
+      @writes_complete = true
+      finished
       raise e
     end
 
@@ -212,6 +230,8 @@ module GRPC
           @readq.push(e)  # let each_queued_msg terminate with this error
         end
         GRPC.logger.debug('bidi-read-loop: finished')
+        @reads_complete = true
+        finished
       end
     end
   end
diff --git a/src/ruby/lib/grpc/generic/client_stub.rb b/src/ruby/lib/grpc/generic/client_stub.rb
index 12946fe819b662aa63f1b30c865c4a74ec2610bb..cddca13d173a6f469723f66ccc6c8e393831bd36 100644
--- a/src/ruby/lib/grpc/generic/client_stub.rb
+++ b/src/ruby/lib/grpc/generic/client_stub.rb
@@ -44,21 +44,21 @@ module GRPC
 
     # setup_channel is used by #initialize to constuct a channel from its
     # arguments.
-    def self.setup_channel(alt_chan, host, creds, **kw)
+    def self.setup_channel(alt_chan, host, creds, channel_args = {})
       unless alt_chan.nil?
         fail(TypeError, '!Channel') unless alt_chan.is_a?(Core::Channel)
         return alt_chan
       end
-      if kw['grpc.primary_user_agent'].nil?
-        kw['grpc.primary_user_agent'] = ''
+      if channel_args['grpc.primary_user_agent'].nil?
+        channel_args['grpc.primary_user_agent'] = ''
       else
-        kw['grpc.primary_user_agent'] += ' '
+        channel_args['grpc.primary_user_agent'] += ' '
       end
-      kw['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
+      channel_args['grpc.primary_user_agent'] += "grpc-ruby/#{VERSION}"
       unless creds.is_a?(Core::ChannelCredentials) || creds.is_a?(Symbol)
         fail(TypeError, '!ChannelCredentials or Symbol')
       end
-      Core::Channel.new(host, kw, creds)
+      Core::Channel.new(host, channel_args, creds)
     end
 
     # Allows users of the stub to modify the propagate mask.
@@ -96,15 +96,16 @@ module GRPC
     #     :this_channel_is_insecure
     # @param channel_override [Core::Channel] a pre-created channel
     # @param timeout [Number] the default timeout to use in requests
-    # @param kw [KeywordArgs]the channel arguments
+    # @param channel_args [Hash] the channel arguments
     def initialize(host, q, creds,
                    channel_override: nil,
                    timeout: nil,
                    propagate_mask: nil,
-                   **kw)
+                   channel_args: {})
       fail(TypeError, '!CompletionQueue') unless q.is_a?(Core::CompletionQueue)
-      @ch = ClientStub.setup_channel(channel_override, host, creds, **kw)
-      alt_host = kw[Core::Channel::SSL_TARGET]
+      @ch = ClientStub.setup_channel(channel_override, host, creds,
+                                     channel_args)
+      alt_host = channel_args[Core::Channel::SSL_TARGET]
       @host = alt_host.nil? ? host : alt_host
       @propagate_mask = propagate_mask
       @timeout = timeout.nil? ? DEFAULT_TIMEOUT : timeout
@@ -135,42 +136,35 @@ module GRPC
     # If return_op is true, the call returns an Operation, calling execute
     # on the Operation returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
+    # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
-    # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object] the response received from the server
     def request_response(method, req, marshal, unmarshal,
                          deadline: nil,
-                         timeout: nil,
                          return_op: false,
                          parent: nil,
                          credentials: nil,
-                         **kw)
+                         metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.request_response(req, **kw) unless return_op
+      return c.request_response(req, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #request_response.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.request_response(req, **kw)
+        c.request_response(req, metadata: metadata)
       end
       op
     end
@@ -205,42 +199,35 @@ module GRPC
     #
     # If return_op is true, the call returns the response.
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # @param method [String] the RPC method to call on the GRPC server
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false] return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @return [Object|Operation] the response received from the server
     def client_streamer(method, requests, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw)
+                        metadata: {})
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.client_streamer(requests, **kw) unless return_op
+      return c.client_streamer(requests, metadata: metadata) unless return_op
 
       # return the operation view of the active_call; define #execute as a
       # new method for this instance that invokes #client_streamer.
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.client_streamer(requests, **kw)
+        c.client_streamer(requests, metadata: metadata)
       end
       op
     end
@@ -292,35 +279,33 @@ module GRPC
     # @param req [Object] the request sent to the server
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param return_op [true|false]return an Operation if true
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|Operation|nil] as discussed above
     def server_streamer(method, req, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         return_op: false,
                         parent: nil,
                         credentials: nil,
-                        **kw,
+                        metadata: {},
                         &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
-      return c.server_streamer(req, **kw, &blk) unless return_op
+      return c.server_streamer(req, metadata: metadata, &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #server_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.server_streamer(req, **kw, &blk)
+        c.server_streamer(req, metadata: metadata, &blk)
       end
       op
     end
@@ -391,11 +376,6 @@ module GRPC
     # * the deadline is exceeded
     #
     #
-    # == Keyword Args ==
-    #
-    # Unspecified keyword arguments are treated as metadata to be sent to the
-    # server.
-    #
     # == Return Value ==
     #
     # if the return_op is false, the return value is an Enumerator of the
@@ -411,36 +391,35 @@ module GRPC
     # @param requests [Object] an Enumerable of requests to send
     # @param marshal [Function] f(obj)->string that marshals requests
     # @param unmarshal [Function] f(string)->obj that unmarshals responses
-    # @param timeout [Numeric] (optional) the max completion time in seconds
     # @param deadline [Time] (optional) the time the request should complete
     # @param parent [Core::Call] a prior call whose reserved metadata
     #   will be propagated by this one.
     # @param credentials [Core::CallCredentials] credentials to use when making
     #   the call
     # @param return_op [true|false] return an Operation if true
+    # @param metadata [Hash] metadata to be sent to the server
     # @param blk [Block] when provided, is executed for each response
     # @return [Enumerator|nil|Operation] as discussed above
     def bidi_streamer(method, requests, marshal, unmarshal,
                       deadline: nil,
-                      timeout: nil,
                       return_op: false,
                       parent: nil,
                       credentials: nil,
-                      **kw,
+                      metadata: {},
                       &blk)
       c = new_active_call(method, marshal, unmarshal,
                           deadline: deadline,
-                          timeout: timeout,
                           parent: parent,
                           credentials: credentials)
 
-      return c.bidi_streamer(requests, **kw, &blk) unless return_op
+      return c.bidi_streamer(requests, metadata: metadata,
+                             &blk) unless return_op
 
       # return the operation view of the active_call; define #execute
       # as a new method for this instance that invokes #bidi_streamer
       op = c.operation
       op.define_singleton_method(:execute) do
-        c.bidi_streamer(requests, **kw, &blk)
+        c.bidi_streamer(requests, metadata: metadata, &blk)
       end
       op
     end
@@ -457,12 +436,10 @@ module GRPC
     # @param timeout [TimeConst]
     def new_active_call(method, marshal, unmarshal,
                         deadline: nil,
-                        timeout: nil,
                         parent: nil,
                         credentials: nil)
-      if deadline.nil?
-        deadline = from_relative_time(timeout.nil? ? @timeout : timeout)
-      end
+
+      deadline = from_relative_time(@timeout) if deadline.nil?
       # Provide each new client call with its own completion queue
       call_queue = Core::CompletionQueue.new
       call = @ch.create_call(call_queue,
diff --git a/src/ruby/lib/grpc/generic/rpc_desc.rb b/src/ruby/lib/grpc/generic/rpc_desc.rb
index cc21ffd3c5b77d715c0521f21f375fa60b00a99b..913f55d0d3ba56552ba712261fb32d2e7e12ce68 100644
--- a/src/ruby/lib/grpc/generic/rpc_desc.rb
+++ b/src/ruby/lib/grpc/generic/rpc_desc.rb
@@ -80,12 +80,12 @@ module GRPC
       else  # is a bidi_stream
         active_call.run_server_bidi(mth)
       end
-      send_status(active_call, OK, 'OK', **active_call.output_metadata)
+      send_status(active_call, OK, 'OK', active_call.output_metadata)
     rescue BadStatus => e
       # this is raised by handlers that want GRPC to send an application error
       # code and detail message and some additional app-specific metadata.
       GRPC.logger.debug("app err:#{active_call}, status:#{e.code}:#{e.details}")
-      send_status(active_call, e.code, e.details, **e.metadata)
+      send_status(active_call, e.code, e.details, e.metadata)
     rescue Core::CallError => e
       # This is raised by GRPC internals but should rarely, if ever happen.
       # Log it, but don't notify the other endpoint..
@@ -135,10 +135,10 @@ module GRPC
       "##{mth.name}: bad arg count; got:#{mth.arity}, want:#{want}, #{msg}"
     end
 
-    def send_status(active_client, code, details, **kw)
+    def send_status(active_client, code, details, metadata = {})
       details = 'Not sure why' if details.nil?
       GRPC.logger.debug("Sending status  #{code}:#{details}")
-      active_client.send_status(code, details, code == OK, **kw)
+      active_client.send_status(code, details, code == OK, metadata: metadata)
     rescue StandardError => e
       GRPC.logger.warn("Could not send status #{code}:#{details}")
       GRPC.logger.warn(e)
diff --git a/src/ruby/lib/grpc/generic/rpc_server.rb b/src/ruby/lib/grpc/generic/rpc_server.rb
index e1496d491af57f9cb08be2424c8e8c41e52e8397..ab7333d1337816029a90aaeff31aeffec70ab900 100644
--- a/src/ruby/lib/grpc/generic/rpc_server.rb
+++ b/src/ruby/lib/grpc/generic/rpc_server.rb
@@ -169,14 +169,6 @@ module GRPC
       alt_cq
     end
 
-    # setup_srv is used by #initialize to constuct a Core::Server from its
-    # arguments.
-    def self.setup_srv(alt_srv, cq, **kw)
-      return Core::Server.new(cq, kw) if alt_srv.nil?
-      fail(TypeError, '!Server') unless alt_srv.is_a? Core::Server
-      alt_srv
-    end
-
     # setup_connect_md_proc is used by #initialize to validate the
     # connect_md_proc.
     def self.setup_connect_md_proc(a_proc)
@@ -193,9 +185,6 @@ module GRPC
     # instance, however other arbitrary are allowed and when present are used
     # to configure the listeninng connection set up by the RpcServer.
     #
-    # * server_override: which if passed must be a [GRPC::Core::Server].  When
-    # present.
-    #
     # * poll_period: when present, the server polls for new events with this
     # period
     #
@@ -217,13 +206,15 @@ module GRPC
     # when non-nil is a proc for determining metadata to to send back the client
     # on receiving an invocation req.  The proc signature is:
     # {key: val, ..} func(method_name, {key: val, ...})
+    #
+    # * server_args:
+    # A server arguments hash to be passed down to the underlying core server
     def initialize(pool_size:DEFAULT_POOL_SIZE,
                    max_waiting_requests:DEFAULT_MAX_WAITING_REQUESTS,
                    poll_period:DEFAULT_POLL_PERIOD,
                    completion_queue_override:nil,
-                   server_override:nil,
                    connect_md_proc:nil,
-                   **kw)
+                   server_args:{})
       @connect_md_proc = RpcServer.setup_connect_md_proc(connect_md_proc)
       @cq = RpcServer.setup_cq(completion_queue_override)
       @max_waiting_requests = max_waiting_requests
@@ -235,7 +226,7 @@ module GRPC
       # running_state can take 4 values: :not_started, :running, :stopping, and
       # :stopped. State transitions can only proceed in that order.
       @running_state = :not_started
-      @server = RpcServer.setup_srv(server_override, @cq, **kw)
+      @server = Core::Server.new(@cq, server_args)
     end
 
     # stops a running server
@@ -364,7 +355,7 @@ module GRPC
       return an_rpc if @pool.jobs_waiting <= @max_waiting_requests
       GRPC.logger.warn("NOT AVAILABLE: too many jobs_waiting: #{an_rpc}")
       noop = proc { |x| x }
-      c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline)
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq, noop, noop, an_rpc.deadline)
       c.send_status(GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED, '')
       nil
     end
@@ -375,7 +366,7 @@ module GRPC
       return an_rpc if rpc_descs.key?(mth)
       GRPC.logger.warn("UNIMPLEMENTED: #{an_rpc}")
       noop = proc { |x| x }
-      c = ActiveCall.new(an_rpc.call, @cq, noop, noop, an_rpc.deadline)
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq, noop, noop, an_rpc.deadline)
       c.send_status(GRPC::Core::StatusCodes::UNIMPLEMENTED, '')
       nil
     end
@@ -386,7 +377,8 @@ module GRPC
       loop_tag = Object.new
       while running_state == :running
         begin
-          an_rpc = @server.request_call(@cq, loop_tag, INFINITE_FUTURE)
+          comp_queue = Core::CompletionQueue.new
+          an_rpc = @server.request_call(comp_queue, loop_tag, INFINITE_FUTURE)
           break if (!an_rpc.nil?) && an_rpc.call.nil?
           active_call = new_active_server_call(an_rpc)
           unless active_call.nil?
@@ -425,15 +417,16 @@ module GRPC
       unless @connect_md_proc.nil?
         connect_md = @connect_md_proc.call(an_rpc.method, an_rpc.metadata)
       end
-      an_rpc.call.run_batch(@cq, handle_call_tag, INFINITE_FUTURE,
+      an_rpc.call.run_batch(an_rpc.cq, handle_call_tag, INFINITE_FUTURE,
                             SEND_INITIAL_METADATA => connect_md)
+
       return nil unless available?(an_rpc)
       return nil unless implemented?(an_rpc)
 
       # Create the ActiveCall
       GRPC.logger.info("deadline is #{an_rpc.deadline}; (now=#{Time.now})")
       rpc_desc = rpc_descs[an_rpc.method.to_sym]
-      c = ActiveCall.new(an_rpc.call, @cq,
+      c = ActiveCall.new(an_rpc.call, an_rpc.cq,
                          rpc_desc.marshal_proc, rpc_desc.unmarshal_proc(:input),
                          an_rpc.deadline)
       mth = an_rpc.method.to_sym
diff --git a/src/ruby/lib/grpc/generic/service.rb b/src/ruby/lib/grpc/generic/service.rb
index 0a166e823e488808e35cbb7b5014ebdc4fc79d02..f30242ee801b7c29d9127447ab0dfb81dc78da8b 100644
--- a/src/ruby/lib/grpc/generic/service.rb
+++ b/src/ruby/lib/grpc/generic/service.rb
@@ -179,24 +179,24 @@ module GRPC
             unmarshal = desc.unmarshal_proc(:output)
             route = "/#{route_prefix}/#{name}"
             if desc.request_response?
-              define_method(mth_name) do |req, **kw|
+              define_method(mth_name) do |req, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                request_response(route, req, marshal, unmarshal, **kw)
+                request_response(route, req, marshal, unmarshal, metadata)
               end
             elsif desc.client_streamer?
-              define_method(mth_name) do |reqs, **kw|
+              define_method(mth_name) do |reqs, metadata = {}|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                client_streamer(route, reqs, marshal, unmarshal, **kw)
+                client_streamer(route, reqs, marshal, unmarshal, metadata)
               end
             elsif desc.server_streamer?
-              define_method(mth_name) do |req, **kw, &blk|
+              define_method(mth_name) do |req, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                server_streamer(route, req, marshal, unmarshal, **kw, &blk)
+                server_streamer(route, req, marshal, unmarshal, metadata, &blk)
               end
             else  # is a bidi_stream
-              define_method(mth_name) do |reqs, **kw, &blk|
+              define_method(mth_name) do |reqs, metadata = {}, &blk|
                 GRPC.logger.debug("calling #{@host}:#{route}")
-                bidi_streamer(route, reqs, marshal, unmarshal, **kw, &blk)
+                bidi_streamer(route, reqs, marshal, unmarshal, metadata, &blk)
               end
             end
           end
diff --git a/src/ruby/pb/test/client.rb b/src/ruby/pb/test/client.rb
index 95b059a18ef6a95bde60396a755fbd2bd222cdba..b6695482a286e8626080e505c1b793bfd9bcef3f 100755
--- a/src/ruby/pb/test/client.rb
+++ b/src/ruby/pb/test/client.rb
@@ -114,7 +114,9 @@ def create_stub(opts)
   if opts.secure
     creds = ssl_creds(opts.use_test_ca)
     stub_opts = {
-      GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      channel_args: {
+        GRPC::Core::Channel::SSL_TARGET => opts.host_override
+      }
     }
 
     # Add service account creds if specified
@@ -315,7 +317,8 @@ class NamedTests
   def timeout_on_sleeping_server
     msg_sizes = [[27_182, 31_415]]
     ppp = PingPongPlayer.new(msg_sizes)
-    resps = @stub.full_duplex_call(ppp.each_item, timeout: 0.001)
+    deadline = GRPC::Core::TimeConsts::from_relative_time(0.001)
+    resps = @stub.full_duplex_call(ppp.each_item, deadline: deadline)
     resps.each { |r| ppp.queue.push(r) }
     fail 'Should have raised GRPC::BadStatus(DEADLINE_EXCEEDED)'
   rescue GRPC::BadStatus => e
diff --git a/src/ruby/qps/client.rb b/src/ruby/qps/client.rb
index d04f707479b3ebbbc8f1b3fa60d6a7cc32dc5eee..917b01271e8666b427fd771367ea8e8bfb0f2439 100644
--- a/src/ruby/qps/client.rb
+++ b/src/ruby/qps/client.rb
@@ -66,8 +66,10 @@ class BenchmarkClient
         cred = GRPC::Core::ChannelCredentials.new()
       end
       if config.security_params.server_host_override
-        opts[GRPC::Core::Channel::SSL_TARGET] =
+        channel_args = {}
+        channel_args[GRPC::Core::Channel::SSL_TARGET] =
           config.security_params.server_host_override
+        opts[:channel_args] = channel_args
       end
     else
       cred = :this_channel_is_insecure
diff --git a/src/ruby/qps/server.rb b/src/ruby/qps/server.rb
index f05fbbdaaf9fe5eb72bfe86d87dac4f0562184c3..52a89ce847c37a4bb80d302622c53578166665b9 100644
--- a/src/ruby/qps/server.rb
+++ b/src/ruby/qps/server.rb
@@ -75,13 +75,14 @@ class BenchmarkServer
     @port = @server.add_http2_port("0.0.0.0:" + port.to_s, cred)
     @server.handle(BenchmarkServiceImpl.new)
     @start_time = Time.now
-    Thread.new {
+    t = Thread.new {
       @server.run
     }
+    t.abort_on_exception
   end
   def mark(reset)
     s = Grpc::Testing::ServerStats.new(time_elapsed:
-                                         (Time.now-@start_time).to_f)
+                                       (Time.now-@start_time).to_f)
     @start_time = Time.now if reset
     s
   end
diff --git a/src/ruby/spec/generic/active_call_spec.rb b/src/ruby/spec/generic/active_call_spec.rb
index c0181e21fb018e3b7ab0967180a94f88eab59431..d9c9780c93b3ba7d94f7b70f0d4606a5d805d27a 100644
--- a/src/ruby/spec/generic/active_call_spec.rb
+++ b/src/ruby/spec/generic/active_call_spec.rb
@@ -159,9 +159,10 @@ describe GRPC::ActiveCall do
   end
 
   describe '#client_invoke' do
-    it 'sends keywords as metadata to the server when the are present' do
+    it 'sends metadata to the server when present' do
       call = make_test_call
-      ActiveCall.client_invoke(call, @client_queue, k1: 'v1', k2: 'v2')
+      metadata = { k1: 'v1', k2: 'v2' }
+      ActiveCall.client_invoke(call, @client_queue, metadata)
       recvd_rpc =  @server.request_call(@server_queue, @server_tag, deadline)
       recvd_call = recvd_rpc.call
       expect(recvd_call).to_not be_nil
diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb
index dd8e2e9f7a2619a338f4ad194ba418ab9120a579..168e7fb7919978cc403b7bc3ab6285b028fbe59e 100644
--- a/src/ruby/spec/generic/client_stub_spec.rb
+++ b/src/ruby/spec/generic/client_stub_spec.rb
@@ -68,15 +68,7 @@ describe 'ClientStub' do
   describe '#new' do
     let(:fake_host) { 'localhost:0' }
     it 'can be created from a host and args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { channel_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -84,7 +76,10 @@ describe 'ClientStub' do
     end
 
     it 'can be created with an channel override' do
-      opts = { a_channel_arg: 'an_arg', channel_override: @ch }
+      opts = {
+        channel_args: { a_channel_arg: 'an_arg' },
+        channel_override: @ch
+      }
       blk = proc do
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
@@ -93,7 +88,10 @@ describe 'ClientStub' do
 
     it 'cannot be created with a bad channel override' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg', channel_override: Object.new }
+        opts = {
+          channel_args: { a_channel_arg: 'an_arg' },
+          channel_override: Object.new
+        }
         GRPC::ClientStub.new(fake_host, @cq, :this_channel_is_insecure, **opts)
       end
       expect(&blk).to raise_error
@@ -101,7 +99,7 @@ describe 'ClientStub' do
 
     it 'cannot be created with bad credentials' do
       blk = proc do
-        opts = { a_channel_arg: 'an_arg' }
+        opts = { channel_args: { a_channel_arg: 'an_arg' } }
         GRPC::ClientStub.new(fake_host, @cq, Object.new, **opts)
       end
       expect(&blk).to raise_error
@@ -111,8 +109,10 @@ describe 'ClientStub' do
       certs = load_test_certs
       blk = proc do
         opts = {
-          GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
-          a_channel_arg: 'an_arg'
+          channel_args: {
+            GRPC::Core::Channel::SSL_TARGET => 'foo.test.google.fr',
+            a_channel_arg: 'an_arg'
+          }
         }
         creds = GRPC::Core::ChannelCredentials.new(certs[0], nil, nil)
         GRPC::ClientStub.new(fake_host, @cq, creds,  **opts)
@@ -172,7 +172,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_response(stub)
         stub.request_response(@method, @sent_msg, noop, noop,
-                              k1: 'v1', k2: 'v2')
+                              metadata: { k1: 'v1', k2: 'v2' })
       end
 
       it_behaves_like 'request response'
@@ -181,7 +181,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.request_response(@method, @sent_msg, noop, noop,
-                                   return_op: true, k1: 'v1', k2: 'v2')
+                                   return_op: true,
+                                   metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -196,7 +197,7 @@ describe 'ClientStub' do
         server_port = create_test_server
         host = "localhost:#{server_port}"
         @stub = GRPC::ClientStub.new(host, @cq, :this_channel_is_insecure)
-        @options = { k1: 'v1', k2: 'v2' }
+        @metadata = { k1: 'v1', k2: 'v2' }
         @sent_msgs = Array.new(3) { |i| 'msg_' + (i + 1).to_s }
         @resp = 'a_reply'
       end
@@ -208,7 +209,7 @@ describe 'ClientStub' do
       end
 
       it 'should send metadata to the server ok' do
-        th = run_client_streamer(@sent_msgs, @resp, @pass, @options)
+        th = run_client_streamer(@sent_msgs, @resp, @pass, **@metadata)
         expect(get_response(@stub)).to eq(@resp)
         th.join
       end
@@ -221,7 +222,7 @@ describe 'ClientStub' do
       end
 
       it 'should raise ArgumentError if metadata contains invalid values' do
-        @options.merge!(k3: 3)
+        @metadata.merge!(k3: 3)
         expect do
           get_response(@stub)
         end.to raise_error(ArgumentError,
@@ -231,7 +232,8 @@ describe 'ClientStub' do
 
     describe 'without a call operation' do
       def get_response(stub)
-        stub.client_streamer(@method, @sent_msgs, noop, noop, @options)
+        stub.client_streamer(@method, @sent_msgs, noop, noop,
+                             metadata: @metadata)
       end
 
       it_behaves_like 'client streaming'
@@ -240,7 +242,7 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_response(stub)
         op = stub.client_streamer(@method, @sent_msgs, noop, noop,
-                                  @options.merge(return_op: true))
+                                  return_op: true, metadata: @metadata)
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         op.execute
       end
@@ -290,7 +292,7 @@ describe 'ClientStub' do
     describe 'without a call operation' do
       def get_responses(stub)
         e = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                 k1: 'v1', k2: 'v2')
+                                 metadata: { k1: 'v1', k2: 'v2' })
         expect(e).to be_a(Enumerator)
         e
       end
@@ -301,7 +303,8 @@ describe 'ClientStub' do
     describe 'via a call operation' do
       def get_responses(stub)
         op = stub.server_streamer(@method, @sent_msg, noop, noop,
-                                  return_op: true, k1: 'v1', k2: 'v2')
+                                  return_op: true,
+                                  metadata: { k1: 'v1', k2: 'v2' })
         expect(op).to be_a(GRPC::ActiveCall::Operation)
         e = op.execute
         expect(e).to be_a(Enumerator)
@@ -383,7 +386,7 @@ describe 'ClientStub' do
         stub = GRPC::ClientStub.new(@host, @cq, :this_channel_is_insecure)
         blk = proc do
           e = stub.bidi_streamer(@method, @sent_msgs, noop, noop,
-                                 timeout: 0.001)
+                                 deadline: from_relative_time(0.001))
           e.collect { |r| r }
         end
         expect(&blk).to raise_error GRPC::BadStatus, /Deadline Exceeded/
diff --git a/src/ruby/spec/generic/rpc_desc_spec.rb b/src/ruby/spec/generic/rpc_desc_spec.rb
index 083632a080fbe3f4b7a6e1688f1b412500c93c28..d2080b7ca2d8b8721f57ce6d4a7520387246854e 100644
--- a/src/ruby/spec/generic/rpc_desc_spec.rb
+++ b/src/ruby/spec/generic/rpc_desc_spec.rb
@@ -56,14 +56,14 @@ describe GRPC::RpcDesc do
     it 'sends the specified status if BadStatus is raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                       {})
+                                                       metadata: {})
       this_desc.run_server_method(@call, method(:bad_status))
     end
 
     it 'sends status UNKNOWN if other StandardErrors are raised' do
       expect(@call).to receive(:remote_read).once.and_return(Object.new)
       expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                        false, {})
+                                                        false, metadata: {})
       this_desc.run_server_method(@call, method(:other_error))
     end
 
@@ -93,7 +93,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         this_desc.run_server_method(@call, method(:fake_reqresp))
       end
     end
@@ -106,13 +106,13 @@ describe GRPC::RpcDesc do
 
       it 'sends the specified status if BadStatus is raised' do
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @client_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
-        expect(@call).to receive(:send_status) .once.with(UNKNOWN, @no_reason,
-                                                          false, {})
+        expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
+                                                         false, metadata: {})
         @client_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -128,7 +128,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).once.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @client_streamer.run_server_method(@call, method(:fake_clstream))
       end
     end
@@ -148,7 +148,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:remote_send).twice.with(@ok_response)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @server_streamer.run_server_method(@call, method(:fake_svstream))
       end
     end
@@ -165,14 +165,14 @@ describe GRPC::RpcDesc do
         e = GRPC::BadStatus.new(@bs_code, 'NOK')
         expect(@call).to receive(:run_server_bidi).and_raise(e)
         expect(@call).to receive(:send_status).once.with(@bs_code, 'NOK', false,
-                                                         {})
+                                                         metadata: {})
         @bidi_streamer.run_server_method(@call, method(:bad_status_alt))
       end
 
       it 'sends status UNKNOWN if other StandardErrors are raised' do
         expect(@call).to receive(:run_server_bidi).and_raise(StandardError)
         expect(@call).to receive(:send_status).once.with(UNKNOWN, @no_reason,
-                                                         false, {})
+                                                         false, metadata: {})
         @bidi_streamer.run_server_method(@call, method(:other_error_alt))
       end
 
@@ -180,7 +180,7 @@ describe GRPC::RpcDesc do
         expect(@call).to receive(:run_server_bidi)
         expect(@call).to receive(:output_metadata).and_return(fake_md)
         expect(@call).to receive(:send_status).once.with(OK, 'OK', true,
-                                                         **fake_md)
+                                                         metadata: fake_md)
         @bidi_streamer.run_server_method(@call, method(:fake_bidistream))
       end
     end
diff --git a/src/ruby/spec/generic/rpc_server_spec.rb b/src/ruby/spec/generic/rpc_server_spec.rb
index 2a42736237de147b6560d32f1576923f8702304a..943502cea2313a6e92972b7dafdc0bfdd93a4118 100644
--- a/src/ruby/spec/generic/rpc_server_spec.rb
+++ b/src/ruby/spec/generic/rpc_server_spec.rb
@@ -99,7 +99,7 @@ class FailingService
   end
 
   def an_rpc(_req, _call)
-    fail GRPC::BadStatus.new(@code, @details, **@md)
+    fail GRPC::BadStatus.new(@code, @details, @md)
   end
 end
 
@@ -137,24 +137,11 @@ describe GRPC::RpcServer do
     @noop = proc { |x| x }
 
     @server_queue = GRPC::Core::CompletionQueue.new
-    server_host = '0.0.0.0:0'
-    @server = GRPC::Core::Server.new(@server_queue, nil)
-    server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-    @host = "localhost:#{server_port}"
-    @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
   end
 
   describe '#new' do
     it 'can be created with just some args' do
-      opts = { a_channel_arg: 'an_arg' }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'can be created with a default deadline' do
-      opts = { a_channel_arg: 'an_arg', deadline: 5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' } }
       blk = proc do
         RpcServer.new(**opts)
       end
@@ -163,7 +150,7 @@ describe GRPC::RpcServer do
 
     it 'can be created with a completion queue override' do
       opts = {
-        a_channel_arg: 'an_arg',
+        server_args: { a_channel_arg: 'an_arg' },
         completion_queue_override: @server_queue
       }
       blk = proc do
@@ -175,7 +162,7 @@ describe GRPC::RpcServer do
     it 'cannot be created with a bad completion queue override' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: Object.new
         }
         RpcServer.new(**opts)
@@ -186,38 +173,20 @@ describe GRPC::RpcServer do
     it 'cannot be created with invalid ServerCredentials' do
       blk = proc do
         opts = {
-          a_channel_arg: 'an_arg',
+          server_args: { a_channel_arg: 'an_arg' },
           creds: Object.new
         }
         RpcServer.new(**opts)
       end
       expect(&blk).to raise_error
     end
-
-    it 'can be created with a server override' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
-      blk = proc do
-        RpcServer.new(**opts)
-      end
-      expect(&blk).not_to raise_error
-    end
-
-    it 'cannot be created with a bad server override' do
-      blk = proc do
-        opts = {
-          a_channel_arg: 'an_arg',
-          server_override: Object.new
-        }
-        RpcServer.new(**opts)
-      end
-      expect(&blk).to raise_error
-    end
   end
 
   describe '#stopped?' do
     before(:each) do
-      opts = { a_channel_arg: 'an_arg', poll_period: 1.5 }
+      opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1.5 }
       @srv = RpcServer.new(**opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'starts out false' do
@@ -245,28 +214,30 @@ describe GRPC::RpcServer do
 
   describe '#running?' do
     it 'starts out false' do
-      opts = { a_channel_arg: 'an_arg', server_override: @server }
+      opts = {
+        server_args: { a_channel_arg: 'an_arg' }
+      }
       r = RpcServer.new(**opts)
       expect(r.running?).to be(false)
     end
 
     it 'is false if run is called with no services registered', server: true do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       expect { r.run }.to raise_error(RuntimeError)
     end
 
     it 'is true after run is called with a registered service' do
       opts = {
-        a_channel_arg: 'an_arg',
-        poll_period: 2.5,
-        server_override: @server
+        server_args: { a_channel_arg: 'an_arg' },
+        poll_period: 2.5
       }
       r = RpcServer.new(**opts)
+      r.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
       r.handle(EchoService)
       t = Thread.new { r.run }
       r.wait_till_running
@@ -278,8 +249,9 @@ describe GRPC::RpcServer do
 
   describe '#handle' do
     before(:each) do
-      @opts = { a_channel_arg: 'an_arg', poll_period: 1 }
+      @opts = { server_args: { a_channel_arg: 'an_arg' }, poll_period: 1 }
       @srv = RpcServer.new(**@opts)
+      @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
     end
 
     it 'raises if #run has already been called' do
@@ -322,11 +294,13 @@ describe GRPC::RpcServer do
     context 'with no connect_metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        server_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @host = "localhost:#{server_port}"
+        @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       end
 
       it 'should return NOT_FOUND status on unknown methods', server: true do
@@ -383,7 +357,8 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        expect(stub.an_rpc(req, k1: 'v1', k2: 'v2')).to be_a(EchoMsg)
+        expect(stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }))
+          .to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
         @srv.stop
@@ -397,8 +372,11 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        timeout = service.delay + 1.0 # wait for long enough
-        resp = stub.an_rpc(req, timeout: timeout, k1: 'v1', k2: 'v2')
+        timeout = service.delay + 1.0
+        deadline = GRPC::Core::TimeConsts.from_relative_time(timeout)
+        resp = stub.an_rpc(req,
+                           deadline: deadline,
+                           metadata: { k1: 'v1', k2: 'v2' })
         expect(resp).to be_a(EchoMsg)
         wanted_md = [{ 'k1' => 'v1', 'k2' => 'v2' }]
         check_md(wanted_md, service.received_md)
@@ -413,7 +391,7 @@ describe GRPC::RpcServer do
         @srv.wait_till_running
         req = EchoMsg.new
         stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         Thread.new do  # cancel the call
           sleep 0.1
           op.cancel
@@ -443,8 +421,7 @@ describe GRPC::RpcServer do
 
       it 'should return RESOURCE_EXHAUSTED on too many jobs', server: true do
         opts = {
-          a_channel_arg: 'an_arg',
-          server_override: @server,
+          server_args: { a_channel_arg: 'an_arg' },
           completion_queue_override: @server_queue,
           pool_size: 1,
           poll_period: 1,
@@ -452,6 +429,8 @@ describe GRPC::RpcServer do
         }
         alt_srv = RpcServer.new(**opts)
         alt_srv.handle(SlowService)
+        alt_port = alt_srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        alt_host = "0.0.0.0:#{alt_port}"
         t = Thread.new { alt_srv.run }
         alt_srv.wait_till_running
         req = EchoMsg.new
@@ -460,7 +439,7 @@ describe GRPC::RpcServer do
         one_failed_as_unavailable = false
         n.times do
           threads << Thread.new do
-            stub = SlowStub.new(@host, :this_channel_is_insecure, **client_opts)
+            stub = SlowStub.new(alt_host, :this_channel_is_insecure)
             begin
               stub.an_rpc(req)
             rescue GRPC::BadStatus => e
@@ -487,12 +466,13 @@ describe GRPC::RpcServer do
       end
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1,
           connect_md_proc: test_md_proc
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should send connect metadata to the client', server: true do
@@ -501,8 +481,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, metadata: { k1: 'v1', k2: 'v2' }, return_op: true)
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         wanted_md = {
@@ -522,11 +502,12 @@ describe GRPC::RpcServer do
     context 'with trailing metadata' do
       before(:each) do
         server_opts = {
-          server_override: @server,
           completion_queue_override: @server_queue,
           poll_period: 1
         }
         @srv = RpcServer.new(**server_opts)
+        alt_port = @srv.add_http2_port('0.0.0.0:0', :this_port_is_insecure)
+        @alt_host = "0.0.0.0:#{alt_port}"
       end
 
       it 'should be added to BadStatus when requests fail', server: true do
@@ -535,7 +516,7 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = FailingStub.new(@host, :this_channel_is_insecure, **client_opts)
+        stub = FailingStub.new(@alt_host, :this_channel_is_insecure)
         blk = proc { stub.an_rpc(req) }
 
         # confirm it raise the expected error
@@ -560,8 +541,8 @@ describe GRPC::RpcServer do
         t = Thread.new { @srv.run }
         @srv.wait_till_running
         req = EchoMsg.new
-        stub = EchoStub.new(@host, :this_channel_is_insecure, **client_opts)
-        op = stub.an_rpc(req, k1: 'v1', k2: 'v2', return_op: true)
+        stub = EchoStub.new(@alt_host, :this_channel_is_insecure)
+        op = stub.an_rpc(req, return_op: true, metadata: { k1: 'v1', k2: 'v2' })
         expect(op.metadata).to be nil
         expect(op.execute).to be_a(EchoMsg)
         expect(op.metadata).to eq(wanted_trailers)
diff --git a/src/ruby/spec/pb/health/checker_spec.rb b/src/ruby/spec/pb/health/checker_spec.rb
index 5523347f45c7db4206d2ea16e10c81656e1f5b1f..f3d121a31ef1b8ac8c5852bec67d5902b1656da5 100644
--- a/src/ruby/spec/pb/health/checker_spec.rb
+++ b/src/ruby/spec/pb/health/checker_spec.rb
@@ -170,17 +170,15 @@ describe Grpc::Health::Checker do
     before(:each) do
       @server_queue = GRPC::Core::CompletionQueue.new
       server_host = '0.0.0.0:0'
-      @server = GRPC::Core::Server.new(@server_queue, nil)
-      server_port = @server.add_http2_port(server_host, :this_port_is_insecure)
-      @host = "localhost:#{server_port}"
-      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
       @client_opts = { channel_override: @ch }
       server_opts = {
-        server_override: @server,
         completion_queue_override: @server_queue,
         poll_period: 1
       }
       @srv = RpcServer.new(**server_opts)
+      server_port = @srv.add_http2_port(server_host, :this_port_is_insecure)
+      @host = "localhost:#{server_port}"
+      @ch = GRPC::Core::Channel.new(@host, nil, :this_channel_is_insecure)
     end
 
     after(:each) do
diff --git a/templates/Makefile.template b/templates/Makefile.template
index 0d5f0ec5b53e5da8a14605e2aba432625c93d90f..079fef65ae893db5281fa176118ba734acc3b0e2 100644
--- a/templates/Makefile.template
+++ b/templates/Makefile.template
@@ -1207,7 +1207,7 @@
   $(LIBDIR)/$(CONFIG)/pkgconfig/grpc_zookeeper.pc:
   	$(E) "[MAKE]    Generating $@"
   	$(Q) mkdir -p $(@D)
-  	$(Q) echo -e "$(GRPC_ZOOKEEPER_PC_FILE)" >$@
+  	$(Q) echo "$(GRPC_ZOOKEEPER_PC_FILE)" | tr , '\n' >$@
 
   $(LIBDIR)/$(CONFIG)/pkgconfig/grpc++.pc:
   	$(E) "[MAKE]    Generating $@"
@@ -1431,17 +1431,19 @@
   	@echo "Your system looks ready to go."
   	@echo
   else
-  	@echo "We couldn't find protoc 3.0.0+ installed on your system. While this"
-  	@echo "won't prevent grpc from working, you won't be able to compile"
-  	@echo "and run any meaningful code with it."
+  	@echo "Warning: it looks like protoc 3.0.0+ isn't installed on your system,"
+  	@echo "which means that you won't be able to compile .proto files for use"
+  	@echo "with gRPC."
   	@echo
+  	@echo "If you are just using pre-compiled protocol buffers, or you otherwise"
+  	@echo "have no need to compile .proto files, you can ignore this."
   	@echo
-  	@echo "Please download and install protobuf 3.0.0+ from:"
+  	@echo "If you do need protobuf for some reason, you can download and install"
+  	@echo "it from:"
   	@echo
   	@echo "   https://github.com/google/protobuf/releases"
   	@echo
-  	@echo "Once you've done so, or if you think this message is in error,"
-  	@echo "you can re-run this check by doing:"
+  	@echo "Once you've done so, you can re-run this check by doing:"
   	@echo
   	@echo "   make verify-install"
   endif
diff --git a/templates/tools/dockerfile/test/sanity/Dockerfile.template b/templates/tools/dockerfile/test/sanity/Dockerfile.template
index 8e2140e648bb2ebca5d36271a30d958f1d01051f..9987e3526085634e228568ecdc1ad4536f44e27f 100644
--- a/templates/tools/dockerfile/test/sanity/Dockerfile.template
+++ b/templates/tools/dockerfile/test/sanity/Dockerfile.template
@@ -29,7 +29,7 @@
   # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
   # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
   
-  FROM debian:jessie
+  FROM ubuntu:15.10
   
   <%include file="../../apt_get_basic.include"/>
   #========================
@@ -46,10 +46,8 @@
   
   #======================================
   # More sanity test dependencies (bazel)
-  RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" > \
-    /etc/apt/sources.list.d/backports.list
-  RUN apt-get update
-  RUN apt-get -t jessie-backports install -y openjdk-8-jdk
+  RUN apt-get install -y openjdk-8-jdk
+  # TOOD(jtattermusch): pin the bazel version
   RUN git clone https://github.com/bazelbuild/bazel.git /bazel
   RUN cd /bazel && ./compile.sh
   RUN ln -s /bazel/output/bazel /bin/
@@ -58,7 +56,6 @@
   # Docker "inception"
   # Note this is quite the ugly hack.
   # This makes sure that the docker binary we inject has its dependencies.
-  RUN apt-get install libsystemd-journal0
   RUN curl https://get.docker.com/ | sh
   RUN apt-get remove --purge -y docker-engine
   
diff --git a/test/core/bad_client/bad_client.c b/test/core/bad_client/bad_client.c
index e5820688ef143110dd7422f1adc8e466b5c31f3c..f753b6fd2436a07c08d95f430eafcbd53037c884 100644
--- a/test/core/bad_client/bad_client.c
+++ b/test/core/bad_client/bad_client.c
@@ -70,7 +70,7 @@ static void done_write(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
 static void server_setup_transport(void *ts, grpc_transport *transport) {
   thd_args *a = ts;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_server_setup_transport(&exec_ctx, a->server, transport,
+  grpc_server_setup_transport(&exec_ctx, a->server, transport, NULL,
                               grpc_server_get_channel_args(a->server));
   grpc_exec_ctx_finish(&exec_ctx);
 }
diff --git a/test/core/client_config/set_initial_connect_string_test.c b/test/core/client_config/set_initial_connect_string_test.c
index c1b84528663a40a381064fbdad7f58fc361fade1..5390c4ab986b19efaa51bce7ce3015e0c75db7a2 100644
--- a/test/core/client_config/set_initial_connect_string_test.c
+++ b/test/core/client_config/set_initial_connect_string_test.c
@@ -41,7 +41,7 @@
 
 #include "src/core/ext/client_config/initial_connect_string.h"
 #include "src/core/lib/iomgr/sockaddr.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "src/core/lib/support/string.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
@@ -69,6 +69,8 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
   GPR_ASSERT(success);
   gpr_slice_buffer_move_into(&state.temp_incoming_buffer,
                              &state.incoming_buffer);
+  gpr_log(GPR_DEBUG, "got %d bytes, magic is %d bytes",
+          state.incoming_buffer.length, strlen(magic_connect_string));
   if (state.incoming_buffer.length > strlen(magic_connect_string)) {
     gpr_atm_rel_store(&state.done_atm, 1);
     grpc_endpoint_shutdown(exec_ctx, state.tcp);
@@ -80,6 +82,7 @@ static void handle_read(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
 }
 
 static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_pollset *accepting_pollset,
                        grpc_tcp_server_acceptor *acceptor) {
   test_tcp_server *server = arg;
   grpc_closure_init(&on_read, handle_read, NULL);
@@ -117,7 +120,6 @@ static gpr_timespec n_sec_deadline(int seconds) {
 }
 
 static void start_rpc(int use_creds, int target_port) {
-  gpr_atm_rel_store(&state.done_atm, 0);
   state.cq = grpc_completion_queue_create(NULL);
   if (use_creds) {
     state.creds = grpc_fake_transport_security_credentials_create();
@@ -166,8 +168,15 @@ typedef struct {
 static void actually_poll_server(void *arg) {
   poll_args *pa = arg;
   gpr_timespec deadline = n_sec_deadline(10);
-  while (gpr_atm_acq_load(&state.done_atm) == 0 &&
-         gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0) {
+  while (true) {
+    bool done = gpr_atm_acq_load(&state.done_atm) != 0;
+    gpr_timespec time_left =
+        gpr_time_sub(deadline, gpr_now(GPR_CLOCK_REALTIME));
+    gpr_log(GPR_DEBUG, "done=%d, time_left=%d.%09d", done, time_left.tv_sec,
+            time_left.tv_nsec);
+    if (done || gpr_time_cmp(time_left, gpr_time_0(GPR_TIMESPAN)) < 0) {
+      break;
+    }
     test_tcp_server_poll(pa->server, 1);
   }
   gpr_event_set(pa->signal_when_done, (void *)1);
@@ -176,6 +185,7 @@ static void actually_poll_server(void *arg) {
 
 static void poll_server_until_read_done(test_tcp_server *server,
                                         gpr_event *signal_when_done) {
+  gpr_atm_rel_store(&state.done_atm, 0);
   gpr_thd_id id;
   poll_args *pa = gpr_malloc(sizeof(*pa));
   pa->server = server;
diff --git a/test/core/end2end/fixtures/h2_fakesec.c b/test/core/end2end/fixtures/h2_fakesec.c
index 246619bf685ebaaa3bd0a35c50615adb4aa0b0d5..44408b28afa9916d57be0d89d89b5d9809605110 100644
--- a/test/core/end2end/fixtures/h2_fakesec.c
+++ b/test/core/end2end/fixtures/h2_fakesec.c
@@ -40,7 +40,7 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/end2end/fixtures/h2_oauth2.c b/test/core/end2end/fixtures/h2_oauth2.c
index 550ff331408932c058d6b1d0fef689c177f308f6..fc56998cdb42ac9914c72f542cab107d999d77d1 100644
--- a/test/core/end2end/fixtures/h2_oauth2.c
+++ b/test/core/end2end/fixtures/h2_oauth2.c
@@ -41,7 +41,7 @@
 #include <grpc/support/log.h>
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/iomgr/iomgr.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "test/core/end2end/data/ssl_test_data.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/end2end/fixtures/h2_sockpair+trace.c b/test/core/end2end/fixtures/h2_sockpair+trace.c
index 87533a9b7f37ea2ae2ce494f18678e3988139989..6b0769b6088df424958896752241d103618c729a 100644
--- a/test/core/end2end/fixtures/h2_sockpair+trace.c
+++ b/test/core/end2end/fixtures/h2_sockpair+trace.c
@@ -50,6 +50,7 @@
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
@@ -60,7 +61,9 @@
 static void server_setup_transport(void *ts, grpc_transport *transport) {
   grpc_end2end_test_fixture *f = ts;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_server_setup_transport(&exec_ctx, f->server, transport,
+  grpc_endpoint_pair *sfd = f->fixture_data;
+  grpc_endpoint_add_to_pollset(&exec_ctx, sfd->server, grpc_cq_pollset(f->cq));
+  grpc_server_setup_transport(&exec_ctx, f->server, transport, NULL,
                               grpc_server_get_channel_args(f->server));
   grpc_exec_ctx_finish(&exec_ctx);
 }
diff --git a/test/core/end2end/fixtures/h2_sockpair.c b/test/core/end2end/fixtures/h2_sockpair.c
index f28147cf40a99b5e35ddb36d1c040cee551eb43a..7be88f8a68e1be10cc16bf6d7b80e995090525c0 100644
--- a/test/core/end2end/fixtures/h2_sockpair.c
+++ b/test/core/end2end/fixtures/h2_sockpair.c
@@ -49,6 +49,7 @@
 #include "src/core/lib/iomgr/endpoint_pair.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
@@ -59,7 +60,9 @@
 static void server_setup_transport(void *ts, grpc_transport *transport) {
   grpc_end2end_test_fixture *f = ts;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_server_setup_transport(&exec_ctx, f->server, transport,
+  grpc_endpoint_pair *sfd = f->fixture_data;
+  grpc_endpoint_add_to_pollset(&exec_ctx, sfd->server, grpc_cq_pollset(f->cq));
+  grpc_server_setup_transport(&exec_ctx, f->server, transport, NULL,
                               grpc_server_get_channel_args(f->server));
   grpc_exec_ctx_finish(&exec_ctx);
 }
diff --git a/test/core/end2end/fixtures/h2_sockpair_1byte.c b/test/core/end2end/fixtures/h2_sockpair_1byte.c
index 302b16b372dd2dc037cdad523f8f206f2c30c69f..166654bcbfd21f4bbe878314b0305d3415d91f2e 100644
--- a/test/core/end2end/fixtures/h2_sockpair_1byte.c
+++ b/test/core/end2end/fixtures/h2_sockpair_1byte.c
@@ -49,6 +49,7 @@
 #include "src/core/lib/iomgr/endpoint_pair.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/surface/channel.h"
+#include "src/core/lib/surface/completion_queue.h"
 #include "src/core/lib/surface/server.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
@@ -59,7 +60,9 @@
 static void server_setup_transport(void *ts, grpc_transport *transport) {
   grpc_end2end_test_fixture *f = ts;
   grpc_exec_ctx exec_ctx = GRPC_EXEC_CTX_INIT;
-  grpc_server_setup_transport(&exec_ctx, f->server, transport,
+  grpc_endpoint_pair *sfd = f->fixture_data;
+  grpc_endpoint_add_to_pollset(&exec_ctx, sfd->server, grpc_cq_pollset(f->cq));
+  grpc_server_setup_transport(&exec_ctx, f->server, transport, NULL,
                               grpc_server_get_channel_args(f->server));
   grpc_exec_ctx_finish(&exec_ctx);
 }
diff --git a/test/core/end2end/fixtures/h2_ssl.c b/test/core/end2end/fixtures/h2_ssl.c
index 69f761607461d2ca876c4a88a6afc6d6ffc9a9a5..eb28623264a4f5ee76908cf42a131e1710bb9f73 100644
--- a/test/core/end2end/fixtures/h2_ssl.c
+++ b/test/core/end2end/fixtures/h2_ssl.c
@@ -41,7 +41,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/support/tmpfile.h"
diff --git a/test/core/end2end/fixtures/h2_ssl_cert.c b/test/core/end2end/fixtures/h2_ssl_cert.c
index f4dea2cc01a7b002c04f6f22c83b488949bee690..0fa525175ba558a76bc237902887d797527f2a03 100644
--- a/test/core/end2end/fixtures/h2_ssl_cert.c
+++ b/test/core/end2end/fixtures/h2_ssl_cert.c
@@ -41,7 +41,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/support/tmpfile.h"
diff --git a/test/core/end2end/fixtures/h2_ssl_proxy.c b/test/core/end2end/fixtures/h2_ssl_proxy.c
index 151a86cb8f5aee04783754d0fb28b0e6f0c3eedc..238e6bca4611b8d4f66450677b3c74de128b814c 100644
--- a/test/core/end2end/fixtures/h2_ssl_proxy.c
+++ b/test/core/end2end/fixtures/h2_ssl_proxy.c
@@ -41,7 +41,7 @@
 #include <grpc/support/log.h>
 
 #include "src/core/lib/channel/channel_args.h"
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/support/tmpfile.h"
diff --git a/test/core/end2end/fixtures/proxy.c b/test/core/end2end/fixtures/proxy.c
index f6e01ec41c5a5a236434072694bd9f1aeac5a81a..7f10649fa696cadf7cd4b13feae6f5b50a5023d8 100644
--- a/test/core/end2end/fixtures/proxy.c
+++ b/test/core/end2end/fixtures/proxy.c
@@ -282,6 +282,8 @@ static void on_p2s_recv_msg(void *arg, int success) {
     err = grpc_call_start_batch(pc->c2p, &op, 1,
                                 new_closure(on_c2p_sent_message, pc), NULL);
     GPR_ASSERT(err == GRPC_CALL_OK);
+  } else {
+    grpc_byte_buffer_destroy(pc->p2s_msg);
   }
   unrefpc(pc, "on_p2s_recv_msg");
 }
diff --git a/test/core/end2end/fuzzers/api_fuzzer.c b/test/core/end2end/fuzzers/api_fuzzer.c
index cdfa9606236ee58b75f1813edba78a7762076d8a..e6cd88acee43c74b6d11ba8525462f08ab0717a1 100644
--- a/test/core/end2end/fuzzers/api_fuzzer.c
+++ b/test/core/end2end/fuzzers/api_fuzzer.c
@@ -252,7 +252,7 @@ static void do_connect(grpc_exec_ctx *exec_ctx, void *arg, bool success) {
 
     grpc_transport *transport =
         grpc_create_chttp2_transport(exec_ctx, NULL, server, 0);
-    grpc_server_setup_transport(exec_ctx, g_server, transport, NULL);
+    grpc_server_setup_transport(exec_ctx, g_server, transport, NULL, NULL);
     grpc_chttp2_transport_start_reading(exec_ctx, transport, NULL, 0);
 
     grpc_exec_ctx_enqueue(exec_ctx, fc->closure, false, NULL);
@@ -424,15 +424,19 @@ static void add_to_free(call_state *call, void *p) {
 static void read_metadata(input_stream *inp, size_t *count,
                           grpc_metadata **metadata, call_state *cs) {
   *count = next_byte(inp);
-  *metadata = gpr_malloc(*count * sizeof(**metadata));
-  memset(*metadata, 0, *count * sizeof(**metadata));
-  for (size_t i = 0; i < *count; i++) {
-    (*metadata)[i].key = read_string(inp);
-    read_buffer(inp, (char **)&(*metadata)[i].value,
-                &(*metadata)[i].value_length);
-    (*metadata)[i].flags = read_uint32(inp);
-    add_to_free(cs, (void *)(*metadata)[i].key);
-    add_to_free(cs, (void *)(*metadata)[i].value);
+  if (*count) {
+    *metadata = gpr_malloc(*count * sizeof(**metadata));
+    memset(*metadata, 0, *count * sizeof(**metadata));
+    for (size_t i = 0; i < *count; i++) {
+      (*metadata)[i].key = read_string(inp);
+      read_buffer(inp, (char **)&(*metadata)[i].value,
+                  &(*metadata)[i].value_length);
+      (*metadata)[i].flags = read_uint32(inp);
+      add_to_free(cs, (void *)(*metadata)[i].key);
+      add_to_free(cs, (void *)(*metadata)[i].value);
+    }
+  } else {
+    *metadata = gpr_malloc(1);
   }
   add_to_free(cs, *metadata);
 }
diff --git a/test/core/end2end/fuzzers/server_fuzzer.c b/test/core/end2end/fuzzers/server_fuzzer.c
index 40273711ab7ef9d09b61ed8e99ad63b9e8a5468a..0a7d6d92aacc17215e094a9971e1c78931c1b968 100644
--- a/test/core/end2end/fuzzers/server_fuzzer.c
+++ b/test/core/end2end/fuzzers/server_fuzzer.c
@@ -69,7 +69,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   grpc_server_start(server);
   grpc_transport *transport =
       grpc_create_chttp2_transport(&exec_ctx, NULL, mock_endpoint, 0);
-  grpc_server_setup_transport(&exec_ctx, server, transport, NULL);
+  grpc_server_setup_transport(&exec_ctx, server, transport, NULL, NULL);
   grpc_chttp2_transport_start_reading(&exec_ctx, transport, NULL, 0);
 
   grpc_call *call1 = NULL;
diff --git a/test/core/end2end/tests/call_creds.c b/test/core/end2end/tests/call_creds.c
index b555bea740b60e64281bf67f33bac62de74306e1..5c6791f6f78e34e7319f72c791402e670d1704b0 100644
--- a/test/core/end2end/tests/call_creds.c
+++ b/test/core/end2end/tests/call_creds.c
@@ -42,7 +42,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include <grpc/support/useful.h>
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/string.h"
 #include "test/core/end2end/cq_verifier.h"
 
diff --git a/test/core/end2end/tests/request_with_flags.c b/test/core/end2end/tests/request_with_flags.c
index 779895b97014faee9e1a94e186feef026a7d9eaa..f5a8d4d60cf3fd147a6e565d0b5710f34be132ef 100644
--- a/test/core/end2end/tests/request_with_flags.c
+++ b/test/core/end2end/tests/request_with_flags.c
@@ -105,7 +105,7 @@ static void test_invoke_request_with_flags(
   gpr_slice request_payload_slice = gpr_slice_from_copied_string("hello world");
   grpc_byte_buffer *request_payload =
       grpc_raw_byte_buffer_create(&request_payload_slice, 1);
-  gpr_timespec deadline = GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10);
+  gpr_timespec deadline = five_seconds_time();
   grpc_end2end_test_fixture f =
       begin_test(config, "test_invoke_request_with_flags", NULL, NULL);
   cq_verifier *cqv = cq_verifier_create(f.cq);
diff --git a/test/core/iomgr/tcp_server_posix_test.c b/test/core/iomgr/tcp_server_posix_test.c
index 266d2396afd62f7b70571b9edf516da911603bad..222ae774fc739c0f151a8b546826d6ce0c467d32 100644
--- a/test/core/iomgr/tcp_server_posix_test.c
+++ b/test/core/iomgr/tcp_server_posix_test.c
@@ -113,6 +113,7 @@ static void server_weak_ref_set(server_weak_ref *weak_ref,
 }
 
 static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_pollset *pollset,
                        grpc_tcp_server_acceptor *acceptor) {
   grpc_endpoint_shutdown(exec_ctx, tcp);
   grpc_endpoint_destroy(exec_ctx, tcp);
diff --git a/test/core/security/auth_context_test.c b/test/core/security/auth_context_test.c
index d1ead162359d43ff802b42e5e668e33d3d693b8a..e2f44ebe249873c6fb1bd0ca95d322e83cca65bb 100644
--- a/test/core/security/auth_context_test.c
+++ b/test/core/security/auth_context_test.c
@@ -33,7 +33,7 @@
 
 #include <string.h>
 
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 #include "src/core/lib/support/string.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/security/b64_test.c b/test/core/security/b64_test.c
index cea870321d3bc2f2dc6c4c5e679fb967fa6040ec..b26bd026fdfd6eed655286b46816e2aa47b46670 100644
--- a/test/core/security/b64_test.c
+++ b/test/core/security/b64_test.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/util/b64.h"
 
 #include <string.h>
 
diff --git a/test/core/security/create_jwt.c b/test/core/security/create_jwt.c
index 6d4707f3c7674422e4de381fc5bf9a86ad985ba5..3c36b767d302079121e231237f62d506c7faeca8 100644
--- a/test/core/security/create_jwt.c
+++ b/test/core/security/create_jwt.c
@@ -34,8 +34,7 @@
 #include <stdio.h>
 #include <string.h>
 
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
 #include "src/core/lib/support/load_file.h"
 
 #include <grpc/support/alloc.h>
diff --git a/test/core/security/credentials_test.c b/test/core/security/credentials_test.c
index 786729327872b8a17b5280ab491fd15788dfa626..31e06372b9397e313d586aa2bedd5759c8509e69 100644
--- a/test/core/security/credentials_test.c
+++ b/test/core/security/credentials_test.c
@@ -33,7 +33,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #include <openssl/rsa.h>
 #include <stdlib.h>
@@ -45,7 +45,10 @@
 #include <grpc/support/time.h>
 
 #include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+#include "src/core/lib/security/credentials/google_default/google_default_credentials.h"
+#include "src/core/lib/security/credentials/jwt/jwt_credentials.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/support/tmpfile.h"
diff --git a/test/core/security/fetch_oauth2.c b/test/core/security/fetch_oauth2.c
index bd314e90d8e42347035d77c6acfa163d093d3721..2a102fb139bb4b5b6dee5ce3d0b4ca91023deacc 100644
--- a/test/core/security/fetch_oauth2.c
+++ b/test/core/security/fetch_oauth2.c
@@ -42,7 +42,7 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/load_file.h"
 #include "test/core/security/oauth2_utils.h"
 
diff --git a/test/core/security/json_token_test.c b/test/core/security/json_token_test.c
index 3aee52ee5c6ff6d8fd8c961cf3f99bbee157f8f3..405fe56c46034d585236247aeb0b1eab47ea67bc 100644
--- a/test/core/security/json_token_test.c
+++ b/test/core/security/json_token_test.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
 
 #include <openssl/evp.h>
 #include <string.h>
@@ -42,7 +42,8 @@
 #include <grpc/support/slice.h>
 
 #include "src/core/lib/json/json.h"
-#include "src/core/lib/security/b64.h"
+#include "src/core/lib/security/credentials/oauth2/oauth2_credentials.h"
+#include "src/core/lib/security/util/b64.h"
 #include "test/core/util/test_config.h"
 
 /* This JSON key was generated with the GCE console and revoked immediately.
diff --git a/test/core/security/jwt_verifier_test.c b/test/core/security/jwt_verifier_test.c
index 077f44d1d667a0e6ab72bfb022ff510d630243e0..7f4f4ffadfc4d13dbdca0eac715d06a634c281dc 100644
--- a/test/core/security/jwt_verifier_test.c
+++ b/test/core/security/jwt_verifier_test.c
@@ -31,7 +31,7 @@
  *
  */
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 #include <string.h>
 
@@ -43,8 +43,8 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/http/httpcli.h"
-#include "src/core/lib/security/b64.h"
-#include "src/core/lib/security/json_token.h"
+#include "src/core/lib/security/credentials/jwt/json_token.h"
+#include "src/core/lib/security/util/b64.h"
 #include "test/core/util/test_config.h"
 
 /* This JSON key was generated with the GCE console and revoked immediately.
diff --git a/test/core/security/oauth2_utils.c b/test/core/security/oauth2_utils.c
index 20815d184cdbfb7f34d003caefe29932ff68632b..80d21cc602741c09a1a6e9fbf5fe7bbce2b410b0 100644
--- a/test/core/security/oauth2_utils.c
+++ b/test/core/security/oauth2_utils.c
@@ -42,7 +42,7 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 typedef struct {
   gpr_mu *mu;
diff --git a/test/core/security/oauth2_utils.h b/test/core/security/oauth2_utils.h
index eff98270c8a30894d615dfd05f4059d2bb6480f3..0f4e8857b0fb562b445bfedeb77d6ac2a9a119f5 100644
--- a/test/core/security/oauth2_utils.h
+++ b/test/core/security/oauth2_utils.h
@@ -34,7 +34,7 @@
 #ifndef GRPC_TEST_CORE_SECURITY_OAUTH2_UTILS_H
 #define GRPC_TEST_CORE_SECURITY_OAUTH2_UTILS_H
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/test/core/security/print_google_default_creds_token.c b/test/core/security/print_google_default_creds_token.c
index 99bce4fbdfbae8c3893be69b699c762cebe2ca80..1b7036cf9ea3288f9f10da83496a461be5abad35 100644
--- a/test/core/security/print_google_default_creds_token.c
+++ b/test/core/security/print_google_default_creds_token.c
@@ -42,7 +42,8 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/composite/composite_credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/core/lib/support/string.h"
 
 typedef struct {
diff --git a/test/core/security/secure_endpoint_test.c b/test/core/security/secure_endpoint_test.c
index aeaf38209bb3bc5066ac946e7b8f60776b71fdff..6aba21a98c2977df82b94f0894f6c15eea6449b6 100644
--- a/test/core/security/secure_endpoint_test.c
+++ b/test/core/security/secure_endpoint_test.c
@@ -41,7 +41,7 @@
 #include <grpc/support/log.h>
 #include "src/core/lib/iomgr/endpoint_pair.h"
 #include "src/core/lib/iomgr/iomgr.h"
-#include "src/core/lib/security/secure_endpoint.h"
+#include "src/core/lib/security/transport/secure_endpoint.h"
 #include "src/core/lib/tsi/fake_transport_security.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c
index 1a4e64b30c71a46f1cb12f6f182735233cccd8ac..6106bec9d3f90b7969b60b717d7d40dce2b1ab1c 100644
--- a/test/core/security/security_connector_test.c
+++ b/test/core/security/security_connector_test.c
@@ -40,8 +40,8 @@
 #include <grpc/support/string_util.h>
 #include <grpc/support/useful.h>
 
-#include "src/core/lib/security/security_connector.h"
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/support/env.h"
 #include "src/core/lib/support/string.h"
 #include "src/core/lib/support/tmpfile.h"
diff --git a/test/core/security/verify_jwt.c b/test/core/security/verify_jwt.c
index 2274fe18d82d39e7f96a5564765fa1ec21aba827..ecb873b655c5a399ea9b7693bc30b1b205f1fddd 100644
--- a/test/core/security/verify_jwt.c
+++ b/test/core/security/verify_jwt.c
@@ -42,7 +42,7 @@
 #include <grpc/support/slice.h>
 #include <grpc/support/sync.h>
 
-#include "src/core/lib/security/jwt_verifier.h"
+#include "src/core/lib/security/credentials/jwt/jwt_verifier.h"
 
 typedef struct {
   grpc_pollset *pollset;
diff --git a/test/core/surface/completion_queue_test.c b/test/core/surface/completion_queue_test.c
index d62d5a93b178a7b85161be8f2da36168fb71f613..49a1fc441f0b2c7e12c62dd1878d4429c1c1043f 100644
--- a/test/core/surface/completion_queue_test.c
+++ b/test/core/surface/completion_queue_test.c
@@ -63,6 +63,12 @@ static void test_no_op(void) {
   shutdown_and_destroy(grpc_completion_queue_create(NULL));
 }
 
+static void test_pollset_conversion(void) {
+  grpc_completion_queue *cq = grpc_completion_queue_create(NULL);
+  GPR_ASSERT(grpc_cq_from_pollset(grpc_cq_pollset(cq)) == cq);
+  shutdown_and_destroy(cq);
+}
+
 static void test_wait_empty(void) {
   grpc_completion_queue *cc;
   grpc_event event;
@@ -408,6 +414,7 @@ int main(int argc, char **argv) {
   grpc_test_init(argc, argv);
   grpc_init();
   test_no_op();
+  test_pollset_conversion();
   test_wait_empty();
   test_shutdown_then_next_polling();
   test_shutdown_then_next_with_timeout();
diff --git a/test/core/surface/concurrent_connectivity_test.c b/test/core/surface/concurrent_connectivity_test.c
index 28ddf58cc84535de8e718bf1d358293ca3753982..de9ba8d27b6c6f38f1e2362ad3e3efc45cb15063 100644
--- a/test/core/surface/concurrent_connectivity_test.c
+++ b/test/core/surface/concurrent_connectivity_test.c
@@ -96,6 +96,7 @@ void server_thread(void *vargs) {
 }
 
 static void on_connect(grpc_exec_ctx *exec_ctx, void *vargs, grpc_endpoint *tcp,
+                       grpc_pollset *accepting_pollset,
                        grpc_tcp_server_acceptor *acceptor) {
   struct server_thread_args *args = (struct server_thread_args *)vargs;
   (void)acceptor;
diff --git a/test/core/surface/secure_channel_create_test.c b/test/core/surface/secure_channel_create_test.c
index 80419efce4bf46abe0d620c8cd35d95dcb7769ea..b9525031678eb3957ecf8a79812d1f6b53451ef9 100644
--- a/test/core/surface/secure_channel_create_test.c
+++ b/test/core/surface/secure_channel_create_test.c
@@ -37,8 +37,8 @@
 #include <grpc/grpc_security.h>
 #include <grpc/support/log.h>
 #include "src/core/ext/client_config/resolver_registry.h"
-#include "src/core/lib/security/credentials.h"
-#include "src/core/lib/security/security_connector.h"
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
+#include "src/core/lib/security/transport/security_connector.h"
 #include "src/core/lib/surface/channel.h"
 #include "test/core/util/test_config.h"
 
diff --git a/test/core/surface/server_chttp2_test.c b/test/core/surface/server_chttp2_test.c
index d22c16497213ff6746f36ebca7f253a5a867f8b7..f42ca9f9cdfb87d8847f2743108b36511655e4ac 100644
--- a/test/core/surface/server_chttp2_test.c
+++ b/test/core/surface/server_chttp2_test.c
@@ -37,7 +37,8 @@
 #include <grpc/support/host_port.h>
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
+#include "src/core/lib/security/credentials/fake/fake_credentials.h"
 #include "src/core/lib/tsi/fake_transport_security.h"
 #include "test/core/util/port.h"
 #include "test/core/util/test_config.h"
diff --git a/test/core/surface/server_test.c b/test/core/surface/server_test.c
index 3d2e25379a04235c434f4213dc8e31af661f3584..1e94c5a41b41d301048045ac3cb2966a6fc79e28 100644
--- a/test/core/surface/server_test.c
+++ b/test/core/surface/server_test.c
@@ -67,12 +67,14 @@ void test_register_method_fail(void) {
 
 void test_request_call_on_no_server_cq(void) {
   grpc_completion_queue *cc = grpc_completion_queue_create(NULL);
+  grpc_server *server = grpc_server_create(NULL, NULL);
   GPR_ASSERT(GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE ==
-             grpc_server_request_call(NULL, NULL, NULL, NULL, cc, cc, NULL));
+             grpc_server_request_call(server, NULL, NULL, NULL, cc, cc, NULL));
   GPR_ASSERT(GRPC_CALL_ERROR_NOT_SERVER_COMPLETION_QUEUE ==
-             grpc_server_request_registered_call(NULL, NULL, NULL, NULL, NULL,
+             grpc_server_request_registered_call(server, NULL, NULL, NULL, NULL,
                                                  NULL, cc, cc, NULL));
   grpc_completion_queue_destroy(cc);
+  grpc_server_destroy(server);
 }
 
 void test_bind_server_twice(void) {
diff --git a/test/core/util/reconnect_server.c b/test/core/util/reconnect_server.c
index d408374a0955e7b63c6963d093f96fb438734129..6509cc5b685e0a446b903fcc73dd8ff390024e3f 100644
--- a/test/core/util/reconnect_server.c
+++ b/test/core/util/reconnect_server.c
@@ -71,6 +71,7 @@ static void pretty_print_backoffs(reconnect_server *server) {
 }
 
 static void on_connect(grpc_exec_ctx *exec_ctx, void *arg, grpc_endpoint *tcp,
+                       grpc_pollset *accepting_pollset,
                        grpc_tcp_server_acceptor *acceptor) {
   char *peer;
   char *last_colon;
diff --git a/test/cpp/common/auth_property_iterator_test.cc b/test/cpp/common/auth_property_iterator_test.cc
index 0e43d4e1e0dd5b6c0fd4ba4314615b86ace76b33..66225ff335cf34019eac153b50753f713e95e034 100644
--- a/test/cpp/common/auth_property_iterator_test.cc
+++ b/test/cpp/common/auth_property_iterator_test.cc
@@ -38,7 +38,7 @@
 #include "test/cpp/util/string_ref_helper.h"
 
 extern "C" {
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 }
 
 using ::grpc::testing::ToString;
diff --git a/test/cpp/common/secure_auth_context_test.cc b/test/cpp/common/secure_auth_context_test.cc
index 067361334d911a9fb90d74b66237fae09587a3ca..b131452f7361cc5bb2e236dca4bc1c972a758098 100644
--- a/test/cpp/common/secure_auth_context_test.cc
+++ b/test/cpp/common/secure_auth_context_test.cc
@@ -38,7 +38,7 @@
 #include "test/cpp/util/string_ref_helper.h"
 
 extern "C" {
-#include "src/core/lib/security/security_context.h"
+#include "src/core/lib/security/context/security_context.h"
 }
 
 using grpc::testing::ToString;
diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc
index e3408bff75eda5d32b5ceaf8d69cee69ea6d859c..f52aa52f39ffb261e319885f493fbfb15ce5073a 100644
--- a/test/cpp/end2end/end2end_test.cc
+++ b/test/cpp/end2end/end2end_test.cc
@@ -48,7 +48,7 @@
 #include <grpc/support/time.h>
 #include <gtest/gtest.h>
 
-#include "src/core/lib/security/credentials.h"
+#include "src/core/lib/security/credentials/credentials.h"
 #include "src/proto/grpc/testing/duplicate/echo_duplicate.grpc.pb.h"
 #include "src/proto/grpc/testing/echo.grpc.pb.h"
 #include "test/core/util/port.h"
diff --git a/test/cpp/end2end/hybrid_end2end_test.cc b/test/cpp/end2end/hybrid_end2end_test.cc
index 02043a89d3a51027f8c736f5175d5f2b5af33543..2c05db345b40430f6bc0056060844d4cefeddc50 100644
--- a/test/cpp/end2end/hybrid_end2end_test.cc
+++ b/test/cpp/end2end/hybrid_end2end_test.cc
@@ -207,6 +207,9 @@ class HybridEnd2endTest : public ::testing::Test {
     ServerBuilder builder;
     builder.AddListeningPort(server_address_.str(),
                              grpc::InsecureServerCredentials());
+    // Always add a sync unimplemented service: we rely on having at least one
+    // synchronous method to get a listening cq
+    builder.RegisterService(&unimplemented_service_);
     builder.RegisterService(service1);
     if (service2) {
       builder.RegisterService(service2);
@@ -216,7 +219,7 @@ class HybridEnd2endTest : public ::testing::Test {
     }
     // Create a separate cq for each potential handler.
     for (int i = 0; i < 5; i++) {
-      cqs_.push_back(builder.AddCompletionQueue());
+      cqs_.push_back(builder.AddCompletionQueue(false));
     }
     server_ = builder.BuildAndStart();
   }
@@ -252,6 +255,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EchoRequest send_request;
     EchoResponse recv_response;
     ClientContext cli_ctx;
+    cli_ctx.set_fail_fast(false);
     send_request.set_message("Hello");
     Status recv_status = stub_->Echo(&cli_ctx, send_request, &recv_response);
     EXPECT_EQ(send_request.message(), recv_response.message());
@@ -265,6 +269,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EchoRequest send_request;
     EchoResponse recv_response;
     ClientContext cli_ctx;
+    cli_ctx.set_fail_fast(false);
     send_request.set_message("Hello");
     Status recv_status = stub->Echo(&cli_ctx, send_request, &recv_response);
     EXPECT_EQ(send_request.message() + "_dup", recv_response.message());
@@ -276,6 +281,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EchoResponse recv_response;
     grpc::string expected_message;
     ClientContext cli_ctx;
+    cli_ctx.set_fail_fast(false);
     send_request.set_message("Hello");
     auto stream = stub_->RequestStream(&cli_ctx, &recv_response);
     for (int i = 0; i < 5; i++) {
@@ -292,6 +298,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EchoRequest request;
     EchoResponse response;
     ClientContext context;
+    context.set_fail_fast(false);
     request.set_message("hello");
 
     auto stream = stub_->ResponseStream(&context, request);
@@ -311,6 +318,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EchoRequest request;
     EchoResponse response;
     ClientContext context;
+    context.set_fail_fast(false);
     grpc::string msg("hello");
 
     auto stream = stub_->BidiStream(&context);
@@ -338,6 +346,7 @@ class HybridEnd2endTest : public ::testing::Test {
     EXPECT_TRUE(s.ok());
   }
 
+  grpc::testing::UnimplementedService::Service unimplemented_service_;
   std::vector<std::unique_ptr<ServerCompletionQueue> > cqs_;
   std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
   std::unique_ptr<Server> server_;
diff --git a/test/cpp/end2end/server_builder_plugin_test.cc b/test/cpp/end2end/server_builder_plugin_test.cc
index 87e3709d7d272bbf562af639adebf4a960f4207a..8a74621e5ac58bbdcde7746b8458d9f170cbb240 100644
--- a/test/cpp/end2end/server_builder_plugin_test.cc
+++ b/test/cpp/end2end/server_builder_plugin_test.cc
@@ -189,6 +189,7 @@ class ServerBuilderPluginTest : public ::testing::TestWithParam<bool> {
   void StartServer() {
     grpc::string server_address = "localhost:" + to_string(port_);
     builder_->AddListeningPort(server_address, InsecureServerCredentials());
+    cq_ = builder_->AddCompletionQueue();
     server_ = builder_->BuildAndStart();
     EXPECT_TRUE(builder_->plugins_[PLUGIN_NAME] != nullptr);
   }
@@ -206,6 +207,12 @@ class ServerBuilderPluginTest : public ::testing::TestWithParam<bool> {
     EXPECT_TRUE(plugin != nullptr);
     EXPECT_TRUE(plugin->init_server_is_called());
     EXPECT_TRUE(plugin->finish_is_called());
+    server_->Shutdown();
+    void* tag;
+    bool ok;
+    cq_->Shutdown();
+    while (cq_->Next(&tag, &ok))
+      ;
   }
 
   string to_string(const int number) {
@@ -218,6 +225,7 @@ class ServerBuilderPluginTest : public ::testing::TestWithParam<bool> {
   std::shared_ptr<Channel> channel_;
   std::unique_ptr<ServerBuilder> builder_;
   std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
+  std::unique_ptr<ServerCompletionQueue> cq_;
   std::unique_ptr<Server> server_;
   TestServiceImpl service_;
   int port_;
diff --git a/test/cpp/qps/gen_build_yaml.py b/test/cpp/qps/gen_build_yaml.py
index 6b3329b046b71a86bc8dc76851246d59bfa90acd..34b815144112d432339d21a096a3a0dfcf4554d2 100755
--- a/test/cpp/qps/gen_build_yaml.py
+++ b/test/cpp/qps/gen_build_yaml.py
@@ -44,8 +44,29 @@ sys.path.append(run_tests_root)
 import performance.scenario_config as scenario_config
 
 def _scenario_json_string(scenario_json):
+  # tweak parameters to get fast test times
+  scenario_json['warmup_seconds'] = 1
+  scenario_json['benchmark_seconds'] = 1
   return json.dumps(scenario_config.remove_nonproto_fields(scenario_json))
 
+def threads_of_type(scenario_json, path):
+  d = scenario_json
+  for el in path.split('/'):
+    if el not in d:
+      return 0
+    d = d[el]
+  return d
+
+def guess_cpu(scenario_json):
+  client = threads_of_type(scenario_json, 'client_config/async_client_threads')
+  server = threads_of_type(scenario_json, 'server_config/async_server_threads')
+  # make an arbitrary guess if set to auto-detect
+  # about the size of the jenkins instances we have for unit tests
+  if client == 0: client = 8
+  if server == 0: server = 8
+  return (scenario_json['num_clients'] * client +
+          scenario_json['num_servers'] * server)
+
 print yaml.dump({
   'tests': [
     {
@@ -59,7 +80,7 @@ print yaml.dump({
       'language': 'c++',
       'boringssl': True,
       'defaults': 'boringssl',
-      'cpu_cost': 1000.0,
+      'cpu_cost': guess_cpu(scenario_json),
       'exclude_configs': []
     }
     for scenario_json in scenario_config.CXXLanguage().scenarios()
diff --git a/test/cpp/qps/generic_async_streaming_ping_pong_test.cc b/test/cpp/qps/generic_async_streaming_ping_pong_test.cc
deleted file mode 100644
index ea373ece8419f06e92ec6eaea7ebaf3278cf5f2f..0000000000000000000000000000000000000000
--- a/test/cpp/qps/generic_async_streaming_ping_pong_test.cc
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- *
- * Copyright 2015, Google Inc.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are
- * met:
- *
- *     * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- *     * Redistributions in binary form must reproduce the above
- * copyright notice, this list of conditions and the following disclaimer
- * in the documentation and/or other materials provided with the
- * distribution.
- *     * Neither the name of Google Inc. nor the names of its
- * contributors may be used to endorse or promote products derived from
- * this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- */
-
-#include <set>
-
-#include <grpc/support/log.h>
-
-#include "test/cpp/qps/driver.h"
-#include "test/cpp/qps/report.h"
-#include "test/cpp/util/benchmark_config.h"
-
-namespace grpc {
-namespace testing {
-
-static const int WARMUP = 5;
-static const int BENCHMARK = 5;
-
-static void RunGenericAsyncStreamingPingPong() {
-  gpr_log(GPR_INFO, "Running Generic Async Streaming Ping Pong");
-
-  ClientConfig client_config;
-  client_config.set_client_type(ASYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1);
-  client_config.set_client_channels(1);
-  client_config.set_async_client_threads(1);
-  client_config.set_rpc_type(STREAMING);
-  client_config.mutable_load_params()->mutable_closed_loop();
-  auto bbuf = client_config.mutable_payload_config()->mutable_bytebuf_params();
-  bbuf->set_resp_size(0);
-  bbuf->set_req_size(0);
-
-  ServerConfig server_config;
-  server_config.set_server_type(ASYNC_GENERIC_SERVER);
-  server_config.set_async_server_threads(1);
-  *server_config.mutable_payload_config() = client_config.payload_config();
-
-  const auto result =
-      RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2);
-
-  GetReporter()->ReportQPS(*result);
-  GetReporter()->ReportLatency(*result);
-}
-
-}  // namespace testing
-}  // namespace grpc
-
-int main(int argc, char** argv) {
-  grpc::testing::InitBenchmark(&argc, &argv, true);
-
-  grpc::testing::RunGenericAsyncStreamingPingPong();
-  return 0;
-}
diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc
index c3e72d9b17953050ded384bfd18f34d9aac739de..f94ea0cb499c174c0245c9b941361471df3445ad 100644
--- a/test/cpp/qps/qps_test.cc
+++ b/test/cpp/qps/qps_test.cc
@@ -50,8 +50,8 @@ static void RunQPS() {
 
   ClientConfig client_config;
   client_config.set_client_type(ASYNC_CLIENT);
-  client_config.set_outstanding_rpcs_per_channel(1000);
-  client_config.set_client_channels(8);
+  client_config.set_outstanding_rpcs_per_channel(100);
+  client_config.set_client_channels(64);
   client_config.set_async_client_threads(8);
   client_config.set_rpc_type(STREAMING);
   client_config.mutable_load_params()->mutable_closed_loop();
diff --git a/test/cpp/qps/server_async.cc b/test/cpp/qps/server_async.cc
index 1234542687dd00ac8f650a87e47ec73752ee51c7..1eddb1dc1dbf57b589d3b1940826b37562478bde 100644
--- a/test/cpp/qps/server_async.cc
+++ b/test/cpp/qps/server_async.cc
@@ -131,10 +131,10 @@ class AsyncQpsServerTest : public Server {
     }
   }
   ~AsyncQpsServerTest() {
-    server_->Shutdown();
     for (auto ss = shutdown_state_.begin(); ss != shutdown_state_.end(); ++ss) {
       (*ss)->set_shutdown();
     }
+    server_->Shutdown();
     for (auto thr = threads_.begin(); thr != threads_.end(); thr++) {
       thr->join();
     }
diff --git a/tools/README.md b/tools/README.md
index cb6c22dd836e45b5c3b1e18866b1fc031863d628..d142d4aee287a7993786b52b769e15ba8dcc760e 100644
--- a/tools/README.md
+++ b/tools/README.md
@@ -1,6 +1,6 @@
 buildgen: Template renderer for our build system.
 
-distrib: Scripts to distribute language-specific packages.
+distrib: Scripts to distribute language-specific packages and other distribution-related helper scripts.
 
 dockerfile: Docker files to test gRPC.
 
diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh
index b216a20379eec3acb488b509264e1897c7b674e6..c880fc23a234d4735c64c7a9693f2a101af11943 100755
--- a/tools/codegen/core/gen_nano_proto.sh
+++ b/tools/codegen/core/gen_nano_proto.sh
@@ -136,6 +136,13 @@ readonly PROTO_BASENAME=$(basename $INPUT_PROTO .proto)
 sed -i "s:$PROTO_BASENAME.pb.h:${GRPC_OUTPUT_DIR}/$PROTO_BASENAME.pb.h:g" \
   "$OUTPUT_DIR/$PROTO_BASENAME.pb.c"
 
+# Fix up the include guards such that they pass the check_include_guards.py
+# test. Assumes that the generated files are being placed in gRPC src dir.
+readonly INCLUDE_GUARD_BASE=`echo $GRPC_OUTPUT_DIR | tr [a-z/] [A-Z_] | sed s:^.*SRC_::`
+readonly UC_PROTO_BASENAME=`echo $PROTO_BASENAME | tr [a-z] [A-Z]`
+sed -i "s:PB_${UC_PROTO_BASENAME}_PB_H_INCLUDED:GRPC_${INCLUDE_GUARD_BASE}_${UC_PROTO_BASENAME}_PB_H:g" \
+  "$OUTPUT_DIR/$PROTO_BASENAME.pb.h"
+
 # prepend copyright
 TMPFILE=$(mktemp)
 cat $COPYRIGHT_FILE "$OUTPUT_DIR/$PROTO_BASENAME.pb.c" > $TMPFILE
diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile
index 43b2a0c3d96544e0137dc2df75e47d37e556b3e7..70a32c5586ad37dd6229523e9cb3ec73918b2122 100644
--- a/tools/dockerfile/test/sanity/Dockerfile
+++ b/tools/dockerfile/test/sanity/Dockerfile
@@ -27,7 +27,7 @@
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-FROM debian:jessie
+FROM ubuntu:15.10
 
 # Install Git and basic packages.
 RUN apt-get update && apt-get install -y \
@@ -77,9 +77,8 @@ RUN pip install simplejson mako
 
 #======================================
 # More sanity test dependencies (bazel)
-RUN echo "deb http://httpredir.debian.org/debian jessie-backports main" >   /etc/apt/sources.list.d/backports.list
-RUN apt-get update
-RUN apt-get -t jessie-backports install -y openjdk-8-jdk
+RUN apt-get install -y openjdk-8-jdk
+# TOOD(jtattermusch): pin the bazel version
 RUN git clone https://github.com/bazelbuild/bazel.git /bazel
 RUN cd /bazel && ./compile.sh
 RUN ln -s /bazel/output/bazel /bin/
@@ -88,7 +87,6 @@ RUN ln -s /bazel/output/bazel /bin/
 # Docker "inception"
 # Note this is quite the ugly hack.
 # This makes sure that the docker binary we inject has its dependencies.
-RUN apt-get install libsystemd-journal0
 RUN curl https://get.docker.com/ | sh
 RUN apt-get remove --purge -y docker-engine
 
diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal
index ef6de014adf67390ca4bf3d20dacd6a37d88b7d0..1b386fc293091aca64c3ffef4ac470dc874c104e 100644
--- a/tools/doxygen/Doxyfile.core.internal
+++ b/tools/doxygen/Doxyfile.core.internal
@@ -885,15 +885,24 @@ src/core/ext/transport/chttp2/transport/stream_map.h \
 src/core/ext/transport/chttp2/transport/timeout_encoding.h \
 src/core/ext/transport/chttp2/transport/varint.h \
 src/core/ext/transport/chttp2/alpn/alpn.h \
-src/core/lib/security/auth_filters.h \
-src/core/lib/security/b64.h \
-src/core/lib/security/credentials.h \
-src/core/lib/security/handshake.h \
-src/core/lib/security/json_token.h \
-src/core/lib/security/jwt_verifier.h \
-src/core/lib/security/secure_endpoint.h \
-src/core/lib/security/security_connector.h \
-src/core/lib/security/security_context.h \
+src/core/lib/security/context/security_context.h \
+src/core/lib/security/credentials/composite/composite_credentials.h \
+src/core/lib/security/credentials/credentials.h \
+src/core/lib/security/credentials/fake/fake_credentials.h \
+src/core/lib/security/credentials/google_default/google_default_credentials.h \
+src/core/lib/security/credentials/iam/iam_credentials.h \
+src/core/lib/security/credentials/jwt/json_token.h \
+src/core/lib/security/credentials/jwt/jwt_credentials.h \
+src/core/lib/security/credentials/jwt/jwt_verifier.h \
+src/core/lib/security/credentials/oauth2/oauth2_credentials.h \
+src/core/lib/security/credentials/plugin/plugin_credentials.h \
+src/core/lib/security/credentials/ssl/ssl_credentials.h \
+src/core/lib/security/transport/auth_filters.h \
+src/core/lib/security/transport/handshake.h \
+src/core/lib/security/transport/secure_endpoint.h \
+src/core/lib/security/transport/security_connector.h \
+src/core/lib/security/util/b64.h \
+src/core/lib/security/util/json_util.h \
 src/core/lib/tsi/fake_transport_security.h \
 src/core/lib/tsi/ssl_transport_security.h \
 src/core/lib/tsi/ssl_types.h \
@@ -927,6 +936,7 @@ src/core/ext/load_reporting/load_reporting_filter.h \
 src/core/ext/census/aggregation.h \
 src/core/ext/census/census_interface.h \
 src/core/ext/census/census_rpc_stats.h \
+src/core/ext/census/gen/census.pb.h \
 src/core/ext/census/grpc_filter.h \
 src/core/ext/census/mlog.h \
 src/core/ext/census/rpc_metric_id.h \
@@ -1037,20 +1047,28 @@ src/core/ext/transport/chttp2/transport/varint.c \
 src/core/ext/transport/chttp2/transport/writing.c \
 src/core/ext/transport/chttp2/alpn/alpn.c \
 src/core/lib/http/httpcli_security_connector.c \
-src/core/lib/security/b64.c \
-src/core/lib/security/client_auth_filter.c \
-src/core/lib/security/credentials.c \
-src/core/lib/security/credentials_metadata.c \
-src/core/lib/security/credentials_posix.c \
-src/core/lib/security/credentials_win32.c \
-src/core/lib/security/google_default_credentials.c \
-src/core/lib/security/handshake.c \
-src/core/lib/security/json_token.c \
-src/core/lib/security/jwt_verifier.c \
-src/core/lib/security/secure_endpoint.c \
-src/core/lib/security/security_connector.c \
-src/core/lib/security/security_context.c \
-src/core/lib/security/server_auth_filter.c \
+src/core/lib/security/context/security_context.c \
+src/core/lib/security/credentials/composite/composite_credentials.c \
+src/core/lib/security/credentials/credentials.c \
+src/core/lib/security/credentials/credentials_metadata.c \
+src/core/lib/security/credentials/fake/fake_credentials.c \
+src/core/lib/security/credentials/google_default/credentials_posix.c \
+src/core/lib/security/credentials/google_default/credentials_win32.c \
+src/core/lib/security/credentials/google_default/google_default_credentials.c \
+src/core/lib/security/credentials/iam/iam_credentials.c \
+src/core/lib/security/credentials/jwt/json_token.c \
+src/core/lib/security/credentials/jwt/jwt_credentials.c \
+src/core/lib/security/credentials/jwt/jwt_verifier.c \
+src/core/lib/security/credentials/oauth2/oauth2_credentials.c \
+src/core/lib/security/credentials/plugin/plugin_credentials.c \
+src/core/lib/security/credentials/ssl/ssl_credentials.c \
+src/core/lib/security/transport/client_auth_filter.c \
+src/core/lib/security/transport/handshake.c \
+src/core/lib/security/transport/secure_endpoint.c \
+src/core/lib/security/transport/security_connector.c \
+src/core/lib/security/transport/server_auth_filter.c \
+src/core/lib/security/util/b64.c \
+src/core/lib/security/util/json_util.c \
 src/core/lib/surface/init_secure.c \
 src/core/lib/tsi/fake_transport_security.c \
 src/core/lib/tsi/ssl_transport_security.c \
@@ -1092,6 +1110,7 @@ src/core/ext/resolver/sockaddr/sockaddr_resolver.c \
 src/core/ext/load_reporting/load_reporting.c \
 src/core/ext/load_reporting/load_reporting_filter.c \
 src/core/ext/census/context.c \
+src/core/ext/census/gen/census.pb.c \
 src/core/ext/census/grpc_context.c \
 src/core/ext/census/grpc_filter.c \
 src/core/ext/census/grpc_plugin.c \
diff --git a/tools/jenkins/run_full_performance.sh b/tools/jenkins/run_full_performance.sh
new file mode 100755
index 0000000000000000000000000000000000000000..3feda866f2f339dfe0ae86504455b87eed05594e
--- /dev/null
+++ b/tools/jenkins/run_full_performance.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+# Copyright 2015, Google Inc.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+#     * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+#     * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following disclaimer
+# in the documentation and/or other materials provided with the
+# distribution.
+#     * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived from
+# this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# This script is invoked by Jenkins and runs full performance test suite.
+set -ex
+
+# Enter the gRPC repo root
+cd $(dirname $0)/../..
+
+# run 8core client vs 8core server
+tools/run_tests/run_performance_tests.py \
+    -l c++ csharp node ruby java python go \
+    --netperf \
+    --category all \
+    --bq_result_table performance_test.performance_experiment \
+    --remote_worker_host grpc-performance-server-8core grpc-performance-client-8core grpc-performance-client2-8core \
+    || EXIT_CODE=1
+
+# scalability with 32cores (and upload to a different BQ table)
+tools/run_tests/run_performance_tests.py \
+    -l c++ java csharp go \
+    --netperf \
+    --category scalable \
+    --bq_result_table performance_test.performance_experiment_32core \
+    --remote_worker_host grpc-performance-server-32core grpc-performance-client-32core grpc-performance-client2-32core \
+    || EXIT_CODE=1
+
+exit $EXIT_CODE
+
diff --git a/tools/run_tests/README.md b/tools/run_tests/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..dd727f43091a38d9a31743a1b3297ceb463b32a3
--- /dev/null
+++ b/tools/run_tests/README.md
@@ -0,0 +1,52 @@
+#Overview
+
+This directory contains scripts that facilitate building and running tests. We are using python scripts as entrypoint for our
+tests because that gives us the opportunity to run tests using the same commandline regardless of the platform you are using.
+
+#Unit tests (run_tests.py)
+
+Builds gRPC in given language and runs unit tests. Use `tools/run_tests/run_tests.py --help` for more help.
+
+######Example
+`tools/run_tests/run_tests.py -l csharp -c dbg`
+
+######Useful options (among many others)
+- `--use_docker` Builds a docker container containing all the prerequisites for given language and runs the tests under that container.
+- `--build_only` Only build, do not run the tests.
+
+#Interop tests (run_interop_tests.py)
+
+Runs tests for cross-platform/cross-language interoperability. For more details, see [Interop tests descriptions](/doc/interop-test-descriptions.md)
+The script is also capable of running interop tests for grpc-java and grpc-go, using sources checked out alongside the ones of the grpc repository.
+
+######Example
+`tools/run_tests/run_interop_tests.py -l csharp -s c++ --use_docker` (run interop tests with C# client and C++ server)
+
+#Performance benchmarks (run_performance_tests.py)
+
+Runs predefined benchmark scenarios for given languages. Besides the simple configuration of running all the scenarios locally,
+the script also supports orchestrating test runs with client and server running on different machines and uploading the results
+to BigQuery.
+
+######Example
+`tools/run_tests/run_peformance_tests.py -l c++ node`
+
+######Useful options
+- `--regex` use regex to select particular scenarios to run.
+
+#Stress tests (run_stress_tests.py)
+
+Runs modified interop tests clients and servers under heavy load for an extended period of time to discover potential stability issues.
+The tests are internally using Kubernetes to run the client and server on GKE and upload statistics to BigQuery.
+
+`tools/run_tests/stress_test/run_on_gke.py --gcp_project_id=<google-cloud-platform-project-id> --config_file=<path-to-config-file>` 
+
+The directory `tools/run_tests/stress_test/configs/` contains the config files for several scenarios
+
+#Artifacts & Packages (task_runner.py)
+
+A generalized framework for running predefined tasks based on their labels. We use this to building binary artifacts & distrib packages and testing them)
+
+######Example
+`tools/run_tests/task_runner.py -f python artifact linux x64` (build tasks with labels `python`, `artifact`, `linux`, and `x64`)
+
diff --git a/tools/run_tests/configs.json b/tools/run_tests/configs.json
index bcc4118d2f3a64c553c8414b8e6923b04b10af89..b0839ef026a5af61412b050d94aa568e6680eb48 100644
--- a/tools/run_tests/configs.json
+++ b/tools/run_tests/configs.json
@@ -57,7 +57,7 @@
   {
     "config": "ubsan", 
     "environ": {
-      "UBSAN_OPTIONS": "print_stacktrace=1"
+      "UBSAN_OPTIONS": "halt_on_error=1:print_stacktrace=1"
     }, 
     "timeout_multiplier": 1.5
   }, 
diff --git a/tools/run_tests/performance/scenario_config.py b/tools/run_tests/performance/scenario_config.py
index 77b158f27e14d0d41ba14eeac3129de47378e14f..b55d728d840fa71ed4ad385617b36989af135d74 100644
--- a/tools/run_tests/performance/scenario_config.py
+++ b/tools/run_tests/performance/scenario_config.py
@@ -34,6 +34,7 @@ JAVA_WARMUP_SECONDS=15  # Java needs more warmup time for JIT to kick in.
 BENCHMARK_SECONDS=30
 
 SMOKETEST='smoketest'
+SCALABLE='scalable'
 
 SECURE_SECARGS = {'use_test_ca': True,
                   'server_host_override': 'foo.test.google.fr'}
@@ -181,7 +182,7 @@ class CXXLanguage:
     # TODO(ctiller): add 70% load latency test
     for secure in [True, False]:
       secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = [SMOKETEST] if secure else None
+      smoketest_categories = [SMOKETEST] if secure else []
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
@@ -214,20 +215,21 @@ class CXXLanguage:
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           unconstrained_client='async',
           secure=secure,
-          categories=smoketest_categories)
+          categories=smoketest_categories+[SCALABLE])
 
       yield _ping_pong_scenario(
           'cpp_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           unconstrained_client='async',
-          secure=secure)
+          secure=secure,
+          categories=[SCALABLE])
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           unconstrained_client='async', use_generic_payload=True,
           secure=secure,
-          categories=smoketest_categories)
+          categories=smoketest_categories+[SCALABLE])
 
       yield _ping_pong_scenario(
           'cpp_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
@@ -275,12 +277,13 @@ class CSharpLanguage:
         'csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
         unconstrained_client='async',
-        categories=[SMOKETEST])
+        categories=[SMOKETEST,SCALABLE])
 
     yield _ping_pong_scenario(
         'csharp_protobuf_async_streaming_qps_unconstrained', rpc_type='STREAMING',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async')
+        unconstrained_client='async',
+        categories=[SCALABLE])
 
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_sync_unary_ping_pong', rpc_type='UNARY',
@@ -296,17 +299,20 @@ class CSharpLanguage:
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async', server_language='c++')
+        unconstrained_client='async', server_language='c++',
+        categories=[SCALABLE])
 
     yield _ping_pong_scenario(
         'csharp_to_cpp_protobuf_sync_to_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='SYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='sync', server_language='c++')
+        unconstrained_client='sync', server_language='c++',
+        categories=[SCALABLE])
 
     yield _ping_pong_scenario(
         'cpp_to_csharp_protobuf_async_unary_qps_unconstrained', rpc_type='UNARY',
         client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
-        unconstrained_client='async', client_language='c++')
+        unconstrained_client='async', client_language='c++',
+        categories=[SCALABLE])
 
 
   def __str__(self):
@@ -487,7 +493,7 @@ class JavaLanguage:
   def scenarios(self):
     for secure in [True, False]:
       secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = [SMOKETEST] if secure else None
+      smoketest_categories = [SMOKETEST] if secure else []
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_ping_pong_%s' % secstr, rpc_type='STREAMING',
@@ -520,19 +526,21 @@ class JavaLanguage:
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           unconstrained_client='async',
           secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
-          categories=smoketest_categories)
+          categories=smoketest_categories+[SCALABLE])
 
       yield _ping_pong_scenario(
           'java_protobuf_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_SERVER',
           unconstrained_client='async',
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=[SCALABLE])
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='ASYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           unconstrained_client='async', use_generic_payload=True,
-          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS)
+          secure=secure, warmup_seconds=JAVA_WARMUP_SECONDS,
+          categories=[SCALABLE])
 
       yield _ping_pong_scenario(
           'java_generic_async_streaming_qps_one_server_core_%s' % secstr, rpc_type='STREAMING',
@@ -562,7 +570,7 @@ class GoLanguage:
   def scenarios(self):
     for secure in [True, False]:
       secstr = 'secure' if secure else 'insecure'
-      smoketest_categories = [SMOKETEST] if secure else None
+      smoketest_categories = [SMOKETEST] if secure else []
 
       # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
       # but that's mostly because of lack of better name of the enum value. 
@@ -592,14 +600,15 @@ class GoLanguage:
           client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
           unconstrained_client='async',
           secure=secure,
-          categories=smoketest_categories)
+          categories=smoketest_categories+[SCALABLE])
 
       # unconstrained_client='async' is intended (client uses goroutines)
       yield _ping_pong_scenario(
           'go_protobuf_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='SYNC_CLIENT', server_type='SYNC_SERVER',
           unconstrained_client='async',
-          secure=secure)
+          secure=secure,
+          categories=[SCALABLE])
 
       # unconstrained_client='async' is intended (client uses goroutines)
       # ASYNC_GENERIC_SERVER for Go actually uses a sync streaming server,
@@ -608,7 +617,8 @@ class GoLanguage:
           'go_generic_sync_streaming_qps_unconstrained_%s' % secstr, rpc_type='STREAMING',
           client_type='SYNC_CLIENT', server_type='ASYNC_GENERIC_SERVER',
           unconstrained_client='async', use_generic_payload=True,
-          secure=secure)
+          secure=secure,
+          categories=[SCALABLE])
 
       # TODO(jtattermusch): add scenarios go vs C++ 
 
diff --git a/tools/run_tests/run_performance_tests.py b/tools/run_tests/run_performance_tests.py
index 181d62bf4a32c5fc4b7833f02b318fd41f59af43..f037d0d17d9ec403b61b4122d781acdbf74dc5f1 100755
--- a/tools/run_tests/run_performance_tests.py
+++ b/tools/run_tests/run_performance_tests.py
@@ -73,7 +73,6 @@ class QpsWorkerJob:
 
 def create_qpsworker_job(language, shortname=None,
                          port=10000, remote_host=None):
-  # TODO: support more languages
   cmdline = language.worker_cmdline() + ['--driver_port=%s' % port]
   if remote_host:
     user_at_host = '%s@%s' % (_REMOTE_HOST_USERNAME, remote_host)
@@ -373,7 +372,7 @@ argp.add_argument('-r', '--regex', default='.*', type=str,
 argp.add_argument('--bq_result_table', default=None, type=str,
                   help='Bigquery "dataset.table" to upload results to.')
 argp.add_argument('--category',
-                  choices=['smoketest','all'],
+                  choices=['smoketest','all','scalable'],
                   default='all',
                   help='Select a category of tests to run.')
 argp.add_argument('--netperf',
diff --git a/tools/run_tests/run_tests.py b/tools/run_tests/run_tests.py
index 65596dea7f8136dd38e28c82d0b2a3d0415c07e1..0a5625c3f5bd91337c686d90de8147ee6c7f507d 100755
--- a/tools/run_tests/run_tests.py
+++ b/tools/run_tests/run_tests.py
@@ -892,7 +892,7 @@ for l in languages:
 
 language_make_options=[]
 if any(language.make_options() for language in languages):
-  if len(languages) != 1:
+  if not 'gcov' in args.config and len(languages) != 1:
     print 'languages with custom make options cannot be built simultaneously with other languages'
     sys.exit(1)
   else:
diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json
index a7b519da2fddd02bf437a6efb4f021839c0dab9f..c96665e285e2e4c294a0ff0df56edc2d6a4f3370 100644
--- a/tools/run_tests/sources_and_headers.json
+++ b/tools/run_tests/sources_and_headers.json
@@ -1811,44 +1811,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "async_streaming_ping_pong_test", 
-    "src": [
-      "test/cpp/qps/async_streaming_ping_pong_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "async_unary_ping_pong_test", 
-    "src": [
-      "test/cpp/qps/async_unary_ping_pong_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr", 
@@ -2088,25 +2050,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "generic_async_streaming_ping_pong_test", 
-    "src": [
-      "test/cpp/qps/generic_async_streaming_ping_pong_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr", 
@@ -2452,26 +2395,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_config", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "qps_test", 
-    "src": [
-      "test/cpp/qps/qps_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr", 
@@ -2734,44 +2657,6 @@
     "third_party": false, 
     "type": "target"
   }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "sync_streaming_ping_pong_test", 
-    "src": [
-      "test/cpp/qps/sync_streaming_ping_pong_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
-  {
-    "deps": [
-      "gpr", 
-      "gpr_test_util", 
-      "grpc", 
-      "grpc++", 
-      "grpc++_test_util", 
-      "grpc_test_util", 
-      "qps"
-    ], 
-    "headers": [], 
-    "language": "c++", 
-    "name": "sync_unary_ping_pong_test", 
-    "src": [
-      "test/cpp/qps/sync_unary_ping_pong_test.cc"
-    ], 
-    "third_party": false, 
-    "type": "target"
-  }, 
   {
     "deps": [
       "gpr", 
@@ -5441,13 +5326,15 @@
   {
     "deps": [
       "gpr", 
-      "grpc_base"
+      "grpc_base", 
+      "nanopb"
     ], 
     "headers": [
       "include/grpc/census.h", 
       "src/core/ext/census/aggregation.h", 
       "src/core/ext/census/census_interface.h", 
       "src/core/ext/census/census_rpc_stats.h", 
+      "src/core/ext/census/gen/census.pb.h", 
       "src/core/ext/census/grpc_filter.h", 
       "src/core/ext/census/mlog.h", 
       "src/core/ext/census/rpc_metric_id.h"
@@ -5460,6 +5347,8 @@
       "src/core/ext/census/census_interface.h", 
       "src/core/ext/census/census_rpc_stats.h", 
       "src/core/ext/census/context.c", 
+      "src/core/ext/census/gen/census.pb.c", 
+      "src/core/ext/census/gen/census.pb.h", 
       "src/core/ext/census/grpc_context.c", 
       "src/core/ext/census/grpc_filter.c", 
       "src/core/ext/census/grpc_filter.h", 
@@ -6112,15 +6001,24 @@
       "include/grpc/grpc_cronet.h", 
       "include/grpc/grpc_security.h", 
       "include/grpc/grpc_security_constants.h", 
-      "src/core/lib/security/auth_filters.h", 
-      "src/core/lib/security/b64.h", 
-      "src/core/lib/security/credentials.h", 
-      "src/core/lib/security/handshake.h", 
-      "src/core/lib/security/json_token.h", 
-      "src/core/lib/security/jwt_verifier.h", 
-      "src/core/lib/security/secure_endpoint.h", 
-      "src/core/lib/security/security_connector.h", 
-      "src/core/lib/security/security_context.h"
+      "src/core/lib/security/context/security_context.h", 
+      "src/core/lib/security/credentials/composite/composite_credentials.h", 
+      "src/core/lib/security/credentials/credentials.h", 
+      "src/core/lib/security/credentials/fake/fake_credentials.h", 
+      "src/core/lib/security/credentials/google_default/google_default_credentials.h", 
+      "src/core/lib/security/credentials/iam/iam_credentials.h", 
+      "src/core/lib/security/credentials/jwt/json_token.h", 
+      "src/core/lib/security/credentials/jwt/jwt_credentials.h", 
+      "src/core/lib/security/credentials/jwt/jwt_verifier.h", 
+      "src/core/lib/security/credentials/oauth2/oauth2_credentials.h", 
+      "src/core/lib/security/credentials/plugin/plugin_credentials.h", 
+      "src/core/lib/security/credentials/ssl/ssl_credentials.h", 
+      "src/core/lib/security/transport/auth_filters.h", 
+      "src/core/lib/security/transport/handshake.h", 
+      "src/core/lib/security/transport/secure_endpoint.h", 
+      "src/core/lib/security/transport/security_connector.h", 
+      "src/core/lib/security/util/b64.h", 
+      "src/core/lib/security/util/json_util.h"
     ], 
     "language": "c", 
     "name": "grpc_secure", 
@@ -6129,29 +6027,46 @@
       "include/grpc/grpc_security.h", 
       "include/grpc/grpc_security_constants.h", 
       "src/core/lib/http/httpcli_security_connector.c", 
-      "src/core/lib/security/auth_filters.h", 
-      "src/core/lib/security/b64.c", 
-      "src/core/lib/security/b64.h", 
-      "src/core/lib/security/client_auth_filter.c", 
-      "src/core/lib/security/credentials.c", 
-      "src/core/lib/security/credentials.h", 
-      "src/core/lib/security/credentials_metadata.c", 
-      "src/core/lib/security/credentials_posix.c", 
-      "src/core/lib/security/credentials_win32.c", 
-      "src/core/lib/security/google_default_credentials.c", 
-      "src/core/lib/security/handshake.c", 
-      "src/core/lib/security/handshake.h", 
-      "src/core/lib/security/json_token.c", 
-      "src/core/lib/security/json_token.h", 
-      "src/core/lib/security/jwt_verifier.c", 
-      "src/core/lib/security/jwt_verifier.h", 
-      "src/core/lib/security/secure_endpoint.c", 
-      "src/core/lib/security/secure_endpoint.h", 
-      "src/core/lib/security/security_connector.c", 
-      "src/core/lib/security/security_connector.h", 
-      "src/core/lib/security/security_context.c", 
-      "src/core/lib/security/security_context.h", 
-      "src/core/lib/security/server_auth_filter.c", 
+      "src/core/lib/security/context/security_context.c", 
+      "src/core/lib/security/context/security_context.h", 
+      "src/core/lib/security/credentials/composite/composite_credentials.c", 
+      "src/core/lib/security/credentials/composite/composite_credentials.h", 
+      "src/core/lib/security/credentials/credentials.c", 
+      "src/core/lib/security/credentials/credentials.h", 
+      "src/core/lib/security/credentials/credentials_metadata.c", 
+      "src/core/lib/security/credentials/fake/fake_credentials.c", 
+      "src/core/lib/security/credentials/fake/fake_credentials.h", 
+      "src/core/lib/security/credentials/google_default/credentials_posix.c", 
+      "src/core/lib/security/credentials/google_default/credentials_win32.c", 
+      "src/core/lib/security/credentials/google_default/google_default_credentials.c", 
+      "src/core/lib/security/credentials/google_default/google_default_credentials.h", 
+      "src/core/lib/security/credentials/iam/iam_credentials.c", 
+      "src/core/lib/security/credentials/iam/iam_credentials.h", 
+      "src/core/lib/security/credentials/jwt/json_token.c", 
+      "src/core/lib/security/credentials/jwt/json_token.h", 
+      "src/core/lib/security/credentials/jwt/jwt_credentials.c", 
+      "src/core/lib/security/credentials/jwt/jwt_credentials.h", 
+      "src/core/lib/security/credentials/jwt/jwt_verifier.c", 
+      "src/core/lib/security/credentials/jwt/jwt_verifier.h", 
+      "src/core/lib/security/credentials/oauth2/oauth2_credentials.c", 
+      "src/core/lib/security/credentials/oauth2/oauth2_credentials.h", 
+      "src/core/lib/security/credentials/plugin/plugin_credentials.c", 
+      "src/core/lib/security/credentials/plugin/plugin_credentials.h", 
+      "src/core/lib/security/credentials/ssl/ssl_credentials.c", 
+      "src/core/lib/security/credentials/ssl/ssl_credentials.h", 
+      "src/core/lib/security/transport/auth_filters.h", 
+      "src/core/lib/security/transport/client_auth_filter.c", 
+      "src/core/lib/security/transport/handshake.c", 
+      "src/core/lib/security/transport/handshake.h", 
+      "src/core/lib/security/transport/secure_endpoint.c", 
+      "src/core/lib/security/transport/secure_endpoint.h", 
+      "src/core/lib/security/transport/security_connector.c", 
+      "src/core/lib/security/transport/security_connector.h", 
+      "src/core/lib/security/transport/server_auth_filter.c", 
+      "src/core/lib/security/util/b64.c", 
+      "src/core/lib/security/util/b64.h", 
+      "src/core/lib/security/util/json_util.c", 
+      "src/core/lib/security/util/json_util.h", 
       "src/core/lib/surface/init_secure.c"
     ], 
     "third_party": false, 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 527701525dfbe12b71ec2ef8177044fc6e627069..1ac87e28a43670a7106c593d2df498f7635962b9 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -1939,44 +1939,6 @@
       "windows"
     ]
   }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "async_streaming_ping_pong_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "async_unary_ping_pong_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [], 
     "ci_platforms": [
@@ -2227,25 +2189,6 @@
       "windows"
     ]
   }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "generic_async_streaming_ping_pong_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [], 
     "ci_platforms": [
@@ -2389,25 +2332,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 10, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "qps_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [], 
     "ci_platforms": [
@@ -2549,44 +2473,6 @@
       "posix"
     ]
   }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "sync_streaming_ping_pong_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
-  {
-    "args": [], 
-    "ci_platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ], 
-    "cpu_cost": 1.0, 
-    "exclude_configs": [], 
-    "flaky": false, 
-    "gtest": false, 
-    "language": "c++", 
-    "name": "sync_unary_ping_pong_test", 
-    "platforms": [
-      "linux", 
-      "mac", 
-      "posix"
-    ]
-  }, 
   {
     "args": [], 
     "ci_platforms": [
@@ -24672,7 +24558,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_generic_async_streaming_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24681,7 +24567,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24698,7 +24584,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24707,7 +24593,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24724,7 +24610,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24733,7 +24619,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24750,7 +24636,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24759,7 +24645,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24776,7 +24662,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24785,7 +24671,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24802,7 +24688,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24811,7 +24697,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24828,7 +24714,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24837,7 +24723,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24854,7 +24740,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_secure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": {\"use_test_ca\": true, \"server_host_override\": \"foo.test.google.fr\"}, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24863,7 +24749,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 1, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24880,7 +24766,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_generic_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24889,7 +24775,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24906,7 +24792,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24915,7 +24801,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24932,7 +24818,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24941,7 +24827,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24958,7 +24844,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
+      "'{\"name\": \"cpp_protobuf_sync_unary_ping_pong_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"server_type\": \"SYNC_SERVER\"}, \"client_config\": {\"client_type\": \"SYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 1, \"async_client_threads\": 1, \"outstanding_rpcs_per_channel\": 1, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 1}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24967,7 +24853,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 2, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -24984,7 +24870,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_unary_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"UNARY\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -24993,7 +24879,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -25010,7 +24896,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_protobuf_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"server_type\": \"ASYNC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"simple_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -25019,7 +24905,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -25036,7 +24922,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_unconstrained_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 0, \"core_limit\": 0, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -25045,7 +24931,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 8, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
@@ -25062,7 +24948,7 @@
   {
     "args": [
       "--scenario_json", 
-      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 5, \"benchmark_seconds\": 30, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
+      "'{\"name\": \"cpp_generic_async_streaming_qps_one_server_core_insecure\", \"warmup_seconds\": 1, \"benchmark_seconds\": 1, \"num_servers\": 1, \"server_config\": {\"async_server_threads\": 1, \"core_limit\": 1, \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"server_type\": \"ASYNC_GENERIC_SERVER\"}, \"client_config\": {\"client_type\": \"ASYNC_CLIENT\", \"security_params\": null, \"payload_config\": {\"bytebuf_params\": {\"resp_size\": 0, \"req_size\": 0}}, \"client_channels\": 64, \"async_client_threads\": 0, \"outstanding_rpcs_per_channel\": 100, \"rpc_type\": \"STREAMING\", \"load_params\": {\"closed_loop\": {}}, \"histogram_params\": {\"max_possible\": 60000000000.0, \"resolution\": 0.01}}, \"num_clients\": 0}'"
     ], 
     "boringssl": true, 
     "ci_platforms": [
@@ -25071,7 +24957,7 @@
       "posix", 
       "windows"
     ], 
-    "cpu_cost": 1000.0, 
+    "cpu_cost": 1, 
     "defaults": "boringssl", 
     "exclude_configs": [], 
     "flaky": false, 
diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj
index 46241e253c23e23eabcd92be78f444d69204c7f4..9c3444e0e46edb17f41a8ad5c3c0439e54e56c74 100644
--- a/vsprojects/vcxproj/grpc/grpc.vcxproj
+++ b/vsprojects/vcxproj/grpc/grpc.vcxproj
@@ -394,15 +394,24 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\transport\chttp2\transport\timeout_encoding.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\transport\chttp2\transport\varint.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\transport\chttp2\alpn\alpn.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\auth_filters.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\b64.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\handshake.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\json_token.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\jwt_verifier.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\secure_endpoint.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\security_connector.h" />
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\security_context.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\context\security_context.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\composite\composite_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\fake\fake_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\google_default_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\iam\iam_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\json_token.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_verifier.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\oauth2\oauth2_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\plugin\plugin_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\ssl\ssl_credentials.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\auth_filters.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\handshake.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\secure_endpoint.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\security_connector.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\util\b64.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\util\json_util.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\tsi\fake_transport_security.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\tsi\ssl_transport_security.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\tsi\ssl_types.h" />
@@ -436,6 +445,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\aggregation.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_interface.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_rpc_stats.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\mlog.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\rpc_metric_id.h" />
@@ -655,33 +665,49 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\http\httpcli_security_connector.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\b64.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\context\security_context.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\client_auth_filter.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\composite\composite_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_metadata.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials_metadata.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_posix.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\fake\fake_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_win32.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\credentials_posix.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\google_default_credentials.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\credentials_win32.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\handshake.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\google_default_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\json_token.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\iam\iam_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\jwt_verifier.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\json_token.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\secure_endpoint.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\security_connector.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_verifier.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\security_context.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\oauth2\oauth2_credentials.c">
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\server_auth_filter.c">
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\plugin\plugin_credentials.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\ssl\ssl_credentials.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\client_auth_filter.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\handshake.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\secure_endpoint.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\security_connector.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\server_auth_filter.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\util\b64.c">
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\util\json_util.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\surface\init_secure.c">
     </ClCompile>
@@ -765,6 +791,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\context.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_context.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.c">
diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
index 9668b9a11489edab9928fd40dfa2d32522b7b5f5..bc9e589a55b17134a686a52b47810fb8beb14c31 100644
--- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters
@@ -322,47 +322,71 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\http\httpcli_security_connector.c">
       <Filter>src\core\lib\http</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\b64.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\context\security_context.c">
+      <Filter>src\core\lib\security\context</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\client_auth_filter.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\composite\composite_credentials.c">
+      <Filter>src\core\lib\security\credentials\composite</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials.c">
+      <Filter>src\core\lib\security\credentials</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_metadata.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials_metadata.c">
+      <Filter>src\core\lib\security\credentials</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_posix.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\fake\fake_credentials.c">
+      <Filter>src\core\lib\security\credentials\fake</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials_win32.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\credentials_posix.c">
+      <Filter>src\core\lib\security\credentials\google_default</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\google_default_credentials.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\credentials_win32.c">
+      <Filter>src\core\lib\security\credentials\google_default</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\handshake.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\google_default_credentials.c">
+      <Filter>src\core\lib\security\credentials\google_default</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\json_token.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\iam\iam_credentials.c">
+      <Filter>src\core\lib\security\credentials\iam</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\jwt_verifier.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\json_token.c">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\secure_endpoint.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_credentials.c">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\security_connector.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_verifier.c">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\security_context.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\oauth2\oauth2_credentials.c">
+      <Filter>src\core\lib\security\credentials\oauth2</Filter>
     </ClCompile>
-    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\server_auth_filter.c">
-      <Filter>src\core\lib\security</Filter>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\plugin\plugin_credentials.c">
+      <Filter>src\core\lib\security\credentials\plugin</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\credentials\ssl\ssl_credentials.c">
+      <Filter>src\core\lib\security\credentials\ssl</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\client_auth_filter.c">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\handshake.c">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\secure_endpoint.c">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\security_connector.c">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\transport\server_auth_filter.c">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\util\b64.c">
+      <Filter>src\core\lib\security\util</Filter>
+    </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\lib\security\util\json_util.c">
+      <Filter>src\core\lib\security\util</Filter>
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\lib\surface\init_secure.c">
       <Filter>src\core\lib\surface</Filter>
@@ -487,6 +511,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\context.c">
       <Filter>src\core\ext\census</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.c">
+      <Filter>src\core\ext\census\gen</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_context.c">
       <Filter>src\core\ext\census</Filter>
     </ClCompile>
@@ -893,32 +920,59 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\transport\chttp2\alpn\alpn.h">
       <Filter>src\core\ext\transport\chttp2\alpn</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\auth_filters.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\context\security_context.h">
+      <Filter>src\core\lib\security\context</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\composite\composite_credentials.h">
+      <Filter>src\core\lib\security\credentials\composite</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\credentials.h">
+      <Filter>src\core\lib\security\credentials</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\fake\fake_credentials.h">
+      <Filter>src\core\lib\security\credentials\fake</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\google_default\google_default_credentials.h">
+      <Filter>src\core\lib\security\credentials\google_default</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\iam\iam_credentials.h">
+      <Filter>src\core\lib\security\credentials\iam</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\json_token.h">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_credentials.h">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\b64.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\jwt\jwt_verifier.h">
+      <Filter>src\core\lib\security\credentials\jwt</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\oauth2\oauth2_credentials.h">
+      <Filter>src\core\lib\security\credentials\oauth2</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\handshake.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\plugin\plugin_credentials.h">
+      <Filter>src\core\lib\security\credentials\plugin</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\json_token.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\credentials\ssl\ssl_credentials.h">
+      <Filter>src\core\lib\security\credentials\ssl</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\jwt_verifier.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\auth_filters.h">
+      <Filter>src\core\lib\security\transport</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\secure_endpoint.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\handshake.h">
+      <Filter>src\core\lib\security\transport</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\security_connector.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\secure_endpoint.h">
+      <Filter>src\core\lib\security\transport</Filter>
     </ClInclude>
-    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\security_context.h">
-      <Filter>src\core\lib\security</Filter>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\transport\security_connector.h">
+      <Filter>src\core\lib\security\transport</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\util\b64.h">
+      <Filter>src\core\lib\security\util</Filter>
+    </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\lib\security\util\json_util.h">
+      <Filter>src\core\lib\security\util</Filter>
     </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\lib\tsi\fake_transport_security.h">
       <Filter>src\core\lib\tsi</Filter>
@@ -1019,6 +1073,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_rpc_stats.h">
       <Filter>src\core\ext\census</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.h">
+      <Filter>src\core\ext\census\gen</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.h">
       <Filter>src\core\ext\census</Filter>
     </ClInclude>
@@ -1055,6 +1112,9 @@
     <Filter Include="src\core\ext\census">
       <UniqueIdentifier>{9bf70bd2-f553-11b2-c237-abd148971eea}</UniqueIdentifier>
     </Filter>
+    <Filter Include="src\core\ext\census\gen">
+      <UniqueIdentifier>{4a14dd37-5868-c656-7333-fa80574cbb07}</UniqueIdentifier>
+    </Filter>
     <Filter Include="src\core\ext\client_config">
       <UniqueIdentifier>{003725f8-37fc-80b5-deba-baae32caf915}</UniqueIdentifier>
     </Filter>
@@ -1163,6 +1223,42 @@
     <Filter Include="src\core\lib\security">
       <UniqueIdentifier>{c4661d64-349f-01c1-1ba8-0602f9047595}</UniqueIdentifier>
     </Filter>
+    <Filter Include="src\core\lib\security\context">
+      <UniqueIdentifier>{187b52e3-bc78-6c62-3e68-4eb19a257661}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials">
+      <UniqueIdentifier>{c8af33b1-f786-001d-3e92-140872dc9829}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\composite">
+      <UniqueIdentifier>{197ed135-5f84-9f6a-6751-38dc5e9dd38c}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\fake">
+      <UniqueIdentifier>{6d391299-53d7-ee6a-55aa-d4c46cd86e82}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\google_default">
+      <UniqueIdentifier>{412c7418-e90a-de77-5705-7890ba960911}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\iam">
+      <UniqueIdentifier>{718f826c-994b-7dd4-3042-0e999c5c22ba}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\jwt">
+      <UniqueIdentifier>{ab21bcdf-de99-5838-699a-19ecb0c4aa14}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\oauth2">
+      <UniqueIdentifier>{f47a7a32-3166-b899-3622-f062f372feea}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\plugin">
+      <UniqueIdentifier>{46120bcc-03e3-1aaa-fc61-9cef786bd70c}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\credentials\ssl">
+      <UniqueIdentifier>{9d7802bc-d459-1a9b-3c97-868cddcca1d1}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\transport">
+      <UniqueIdentifier>{b22e611f-8272-9914-24a5-8107ebf51eeb}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="src\core\lib\security\util">
+      <UniqueIdentifier>{fcd7b397-aadd-556a-8aae-0cb7c893fbe0}</UniqueIdentifier>
+    </Filter>
     <Filter Include="src\core\lib\surface">
       <UniqueIdentifier>{a21971fb-304f-da08-b1b2-7bd8df8ac373}</UniqueIdentifier>
     </Filter>
diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
index 82eb644dfce34bf3f5b5f67f516ccfe49ec25108..445e8405825fb06f43ab984c7cc8d3080c3e7301 100644
--- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
+++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj
@@ -409,6 +409,7 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\aggregation.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_interface.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_rpc_stats.h" />
+    <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\mlog.h" />
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\rpc_metric_id.h" />
@@ -692,6 +693,8 @@
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\context.c">
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.c">
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_context.c">
     </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.c">
diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
index 120f1bee37ed24d356cdfe378556bfd64af1f292..286d7d5662948f16e2c22e055c3bb631f3f78aa7 100644
--- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
+++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters
@@ -418,6 +418,9 @@
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\context.c">
       <Filter>src\core\ext\census</Filter>
     </ClCompile>
+    <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.c">
+      <Filter>src\core\ext\census\gen</Filter>
+    </ClCompile>
     <ClCompile Include="$(SolutionDir)\..\src\core\ext\census\grpc_context.c">
       <Filter>src\core\ext\census</Filter>
     </ClCompile>
@@ -896,6 +899,9 @@
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\census_rpc_stats.h">
       <Filter>src\core\ext\census</Filter>
     </ClInclude>
+    <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\gen\census.pb.h">
+      <Filter>src\core\ext\census\gen</Filter>
+    </ClInclude>
     <ClInclude Include="$(SolutionDir)\..\src\core\ext\census\grpc_filter.h">
       <Filter>src\core\ext\census</Filter>
     </ClInclude>
@@ -932,6 +938,9 @@
     <Filter Include="src\core\ext\census">
       <UniqueIdentifier>{3f21cd12-b8b9-18f8-8780-e21bbe2285d0}</UniqueIdentifier>
     </Filter>
+    <Filter Include="src\core\ext\census\gen">
+      <UniqueIdentifier>{dfe53168-57b0-3ac4-d8ba-07fd958cc8f5}</UniqueIdentifier>
+    </Filter>
     <Filter Include="src\core\ext\client_config">
       <UniqueIdentifier>{25fa8af3-0a05-987c-741f-fa8ff9d65d51}</UniqueIdentifier>
     </Filter>