diff --git a/include/grpc/grpc_security.h b/include/grpc/grpc_security.h index fcd09db082464a5510126abeba619b2cfb7bbfc5..28881c3a11ab9773f1d73f52aa056d16284d9acb 100644 --- a/include/grpc/grpc_security.h +++ b/include/grpc/grpc_security.h @@ -147,6 +147,29 @@ GRPC_API grpc_channel_credentials *grpc_google_default_credentials_create(void); #define GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR \ "GRPC_DEFAULT_SSL_ROOTS_FILE_PATH" +/* Results for the SSL roots override callback. */ +typedef enum { + GRPC_SSL_ROOTS_OVERRIDE_OK, + GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY, /* Do not try fallback options. */ + GRPC_SSL_ROOTS_OVERRIDE_FAIL +} grpc_ssl_roots_override_result; + +/* Callback for getting the SSL roots override from the application. + In case of success, *pem_roots_certs must be set to a NULL terminated string + containing the list of PEM encoded root certificates. The ownership is passed + to the core and freed (laster by the core) with gpr_free. + If this function fails and GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment is + set to a valid path, it will override the roots specified this func */ +typedef grpc_ssl_roots_override_result (*grpc_ssl_roots_override_callback)( + char **pem_root_certs); + +/* Setup a callback to override the default TLS/SSL roots. + This function is not thread-safe and must be called at initialization time + before any ssl credentials are created to have the desired side effect. + If GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment is set to a valid path, the + callback will not be called. */ +void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb); + /* Object that holds a private key / certificate chain pair in PEM format. */ typedef struct { /* private_key is the NULL-terminated string containing the PEM encoding of @@ -163,8 +186,9 @@ typedef struct { of the server root certificates. If this parameter is NULL, the implementation will first try to dereference the file pointed by the GRPC_DEFAULT_SSL_ROOTS_FILE_PATH environment variable, and if that fails, - get the roots from a well-known place on disk (in the grpc install - directory). + try to get the roots set by grpc_override_ssl_default_roots. Eventually, + if all these fail, it will try to get the roots from a well-known place on + disk (in the grpc install directory). - pem_key_cert_pair is a pointer on the object containing client's private key and certificate chain. This parameter can be NULL if the client does not have such a key/cert pair. */ diff --git a/src/core/security/security_connector.c b/src/core/security/security_connector.c index 61336a1057df088fc9bfcb740fe4e4de4628010c..bdccbabfea3458a4cd2e912aecebe1cd8d610af3 100644 --- a/src/core/security/security_connector.c +++ b/src/core/security/security_connector.c @@ -61,6 +61,14 @@ static const char *installed_roots_path = INSTALL_PREFIX "/share/grpc/roots.pem"; #endif +/* -- Overridden default roots. -- */ + +static grpc_ssl_roots_override_callback ssl_roots_override_cb = NULL; + +void grpc_set_ssl_roots_override_callback(grpc_ssl_roots_override_callback cb) { + ssl_roots_override_cb = cb; +} + /* -- Cipher suites. -- */ /* Defines the cipher suites that we accept by default. All these cipher suites @@ -595,23 +603,44 @@ static grpc_security_connector_vtable ssl_channel_vtable = { static grpc_security_connector_vtable ssl_server_vtable = { ssl_server_destroy, ssl_server_do_handshake, ssl_server_check_peer}; -static gpr_slice default_pem_root_certs; +static gpr_slice compute_default_pem_root_certs_once(void) { + gpr_slice result = gpr_empty_slice(); -static void init_default_pem_root_certs(void) { /* First try to load the roots from the environment. */ char *default_root_certs_path = gpr_getenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR); - if (default_root_certs_path == NULL) { - default_pem_root_certs = gpr_empty_slice(); - } else { - default_pem_root_certs = gpr_load_file(default_root_certs_path, 0, NULL); + if (default_root_certs_path != NULL) { + result = gpr_load_file(default_root_certs_path, 0, NULL); gpr_free(default_root_certs_path); } + /* Try overridden roots if needed. */ + grpc_ssl_roots_override_result ovrd_res = GRPC_SSL_ROOTS_OVERRIDE_FAIL; + if (GPR_SLICE_IS_EMPTY(result) && ssl_roots_override_cb != NULL) { + char *pem_root_certs = NULL; + ovrd_res = ssl_roots_override_cb(&pem_root_certs); + if (ovrd_res == GRPC_SSL_ROOTS_OVERRIDE_OK) { + GPR_ASSERT(pem_root_certs != NULL); + result = gpr_slice_new(pem_root_certs, strlen(pem_root_certs), gpr_free); + } + } + /* Fall back to installed certs if needed. */ - if (GPR_SLICE_IS_EMPTY(default_pem_root_certs)) { - default_pem_root_certs = gpr_load_file(installed_roots_path, 0, NULL); + if (GPR_SLICE_IS_EMPTY(result) && + ovrd_res != GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY) { + result = gpr_load_file(installed_roots_path, 0, NULL); } + return result; +} + +static gpr_slice default_pem_root_certs; + +static void init_default_pem_root_certs(void) { + default_pem_root_certs = compute_default_pem_root_certs_once(); +} + +gpr_slice grpc_get_default_ssl_roots_for_testing(void) { + return compute_default_pem_root_certs_once(); } size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs) { diff --git a/src/core/security/security_connector.h b/src/core/security/security_connector.h index 2b734109b389422100d81bbe6ae85e5c831093cc..39df7821f0c72c79aab3f11e75d96773e809aeb2 100644 --- a/src/core/security/security_connector.h +++ b/src/core/security/security_connector.h @@ -209,6 +209,9 @@ grpc_security_status grpc_ssl_channel_security_connector_create( /* Gets the default ssl roots. */ size_t grpc_get_default_ssl_roots(const unsigned char **pem_root_certs); +/* Exposed for TESTING ONLY!. */ +gpr_slice grpc_get_default_ssl_roots_for_testing(void); + /* Config for ssl servers. */ typedef struct { unsigned char **pem_private_keys; diff --git a/src/core/transport/chttp2/hpack_encoder.c b/src/core/transport/chttp2/hpack_encoder.c index 89a80d896cd59ac47a9f4fde7c2e6df510eb2632..f30f574d0635c9fc3e7ba445f4fc5edb789b6b1a 100644 --- a/src/core/transport/chttp2/hpack_encoder.c +++ b/src/core/transport/chttp2/hpack_encoder.c @@ -283,7 +283,7 @@ static void emit_lithdr_incidx(grpc_chttp2_hpack_compressor *c, len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); GRPC_CHTTP2_WRITE_VARINT(key_index, 2, 0x40, add_tiny_header_data(st, len_pfx), len_pfx); - GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, 0x00, + GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); add_header_data(st, gpr_slice_ref(value_slice)); } @@ -300,7 +300,7 @@ static void emit_lithdr_noidx(grpc_chttp2_hpack_compressor *c, len_val_len = GRPC_CHTTP2_VARINT_LENGTH((uint32_t)len_val, 1); GRPC_CHTTP2_WRITE_VARINT(key_index, 4, 0x00, add_tiny_header_data(st, len_pfx), len_pfx); - GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, 0x00, + GRPC_CHTTP2_WRITE_VARINT((uint32_t)len_val, 1, huffman_prefix, add_tiny_header_data(st, len_val_len), len_val_len); add_header_data(st, gpr_slice_ref(value_slice)); } diff --git a/src/core/transport/chttp2/internal.h b/src/core/transport/chttp2/internal.h index 6823f8f551b44d61b60a768dc7f27a44641a5530..c611496e7e5793731052805c21f82d0995b119d9 100644 --- a/src/core/transport/chttp2/internal.h +++ b/src/core/transport/chttp2/internal.h @@ -35,6 +35,7 @@ #define GRPC_INTERNAL_CORE_CHTTP2_INTERNAL_H #include <assert.h> +#include <stdbool.h> #include "src/core/iomgr/endpoint.h" #include "src/core/transport/chttp2/frame.h" @@ -67,6 +68,9 @@ typedef enum { GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_PARSING, GRPC_CHTTP2_LIST_CLOSED_WAITING_FOR_WRITING, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT, + /* streams waiting for the outgoing window in the writing path, they will be + * merged to the stalled list or writable list under transport lock. */ + GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT, /** streams that are waiting to start because there are too many concurrent streams on the connection */ GRPC_CHTTP2_LIST_WAITING_FOR_CONCURRENCY, @@ -504,11 +508,11 @@ void grpc_chttp2_publish_reads(grpc_exec_ctx *exec_ctx, grpc_chttp2_transport_global *global, grpc_chttp2_transport_parsing *parsing); -/** Get a writable stream - returns non-zero if there was a stream available */ void grpc_chttp2_list_add_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global *stream_global); +/** Get a writable stream + returns non-zero if there was a stream available */ int grpc_chttp2_list_pop_writable_stream( grpc_chttp2_transport_global *transport_global, grpc_chttp2_transport_writing *transport_writing, @@ -560,9 +564,12 @@ int grpc_chttp2_list_pop_check_read_ops( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); -void grpc_chttp2_list_add_stalled_by_transport( +void grpc_chttp2_list_add_writing_stalled_by_transport( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing); +void grpc_chttp2_list_flush_writing_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, bool is_window_available); + int grpc_chttp2_list_pop_stalled_by_transport( grpc_chttp2_transport_global *transport_global, grpc_chttp2_stream_global **stream_global); diff --git a/src/core/transport/chttp2/stream_lists.c b/src/core/transport/chttp2/stream_lists.c index 273a513e2f30df5b36672a5a90fb18af119c0d45..2f31a47cb3df6a177edcc567f618426532757fc4 100644 --- a/src/core/transport/chttp2/stream_lists.c +++ b/src/core/transport/chttp2/stream_lists.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -313,12 +313,27 @@ int grpc_chttp2_list_pop_check_read_ops( return r; } -void grpc_chttp2_list_add_stalled_by_transport( +void grpc_chttp2_list_add_writing_stalled_by_transport( grpc_chttp2_transport_writing *transport_writing, grpc_chttp2_stream_writing *stream_writing) { stream_list_add(TRANSPORT_FROM_WRITING(transport_writing), STREAM_FROM_WRITING(stream_writing), - GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); + GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT); +} + +void grpc_chttp2_list_flush_writing_stalled_by_transport( + grpc_chttp2_transport_writing *transport_writing, + bool is_window_available) { + grpc_chttp2_stream *stream; + grpc_chttp2_transport *transport = TRANSPORT_FROM_WRITING(transport_writing); + while (stream_list_pop(transport, &stream, + GRPC_CHTTP2_LIST_WRITING_STALLED_BY_TRANSPORT)) { + if (is_window_available) { + grpc_chttp2_list_add_writable_stream(&transport->global, &stream->global); + } else { + stream_list_add(transport, stream, GRPC_CHTTP2_LIST_STALLED_BY_TRANSPORT); + } + } } int grpc_chttp2_list_pop_stalled_by_transport( diff --git a/src/core/transport/chttp2/writing.c b/src/core/transport/chttp2/writing.c index f78adb9e26d8e4a4562b5c16c73a83fa2aa8fe84..095883c66da92e8266068ed282f612a050daeff7 100644 --- a/src/core/transport/chttp2/writing.c +++ b/src/core/transport/chttp2/writing.c @@ -130,8 +130,8 @@ int grpc_chttp2_unlocking_check_writes( GRPC_CHTTP2_STREAM_REF(stream_global, "chttp2_writing"); } } else { - grpc_chttp2_list_add_stalled_by_transport(transport_writing, - stream_writing); + grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, + stream_writing); } } if (stream_global->send_trailing_metadata) { @@ -273,8 +273,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, stream_writing->sent_message = 1; } } else if (transport_writing->outgoing_window == 0) { - grpc_chttp2_list_add_stalled_by_transport(transport_writing, - stream_writing); + grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, + stream_writing); grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); } } @@ -312,8 +312,8 @@ static void finalize_outbuf(grpc_exec_ctx *exec_ctx, /* do nothing - already reffed */ } } else { - grpc_chttp2_list_add_stalled_by_transport(transport_writing, - stream_writing); + grpc_chttp2_list_add_writing_stalled_by_transport(transport_writing, + stream_writing); grpc_chttp2_list_add_written_stream(transport_writing, stream_writing); } } else { @@ -329,6 +329,10 @@ void grpc_chttp2_cleanup_writing( grpc_chttp2_transport_writing *transport_writing) { grpc_chttp2_stream_writing *stream_writing; grpc_chttp2_stream_global *stream_global; + bool is_window_available = transport_writing->outgoing_window > 0; + + grpc_chttp2_list_flush_writing_stalled_by_transport(transport_writing, + is_window_available); while (grpc_chttp2_list_pop_written_stream( transport_global, transport_writing, &stream_global, &stream_writing)) { diff --git a/src/python/grpcio/commands.py b/src/python/grpcio/commands.py index 3e3c72ff6ed162db24249583e6024798b78d1813..98ad2e571d83edcf00badd1f3ed3aa8511f4586e 100644 --- a/src/python/grpcio/commands.py +++ b/src/python/grpcio/commands.py @@ -320,11 +320,11 @@ class BuildExt(build_ext.build_ext): extension.extra_link_args += list(BuildExt.LINK_OPTIONS[compiler]) try: build_ext.build_ext.build_extensions(self) - except KeyboardInterrupt: - raise except Exception as error: - support.diagnose_build_ext_error(self, error, traceback.format_exc()) - raise CommandError("Failed `build_ext` step.") + formatted_exception = traceback.format_exc() + support.diagnose_build_ext_error(self, error, formatted_exception) + raise CommandError( + "Failed `build_ext` step:\n{}".format(formatted_exception)) class Gather(setuptools.Command): diff --git a/src/python/grpcio/support.py b/src/python/grpcio/support.py index 96d9cbf4f371f14f0d069cd95792905aca721c23..33244eb388ee2d5614464b78ab00af7663290d73 100644 --- a/src/python/grpcio/support.py +++ b/src/python/grpcio/support.py @@ -77,10 +77,27 @@ def _expect_compile(compiler, source_string, error_message): .format(error_message)) def diagnose_compile_error(build_ext, error): - """Attempt to run a few test files through the compiler to see if we can - diagnose the reason for the compile failure.""" + """Attempt to diagnose an error during compilation.""" for c_check, message in C_CHECKS.items(): _expect_compile(build_ext.compiler, c_check, message) + python_sources = [ + source for source in build_ext.get_source_files() + if source.startswith('./src/python') and source.endswith('c') + ] + for source in python_sources: + if not os.path.isfile(source): + raise commands.CommandError( + ("Diagnostics found a missing Python extension source file:\n{}\n\n" + "This is usually because the Cython sources haven't been transpiled " + "into C yet and you're building from source.\n" + "Try setting the environment variable " + "`GRPC_PYTHON_BUILD_WITH_CYTHON=1` when invoking `setup.py` or " + "when using `pip`, e.g.:\n\n" + "pip install -rrequirements.txt\n" + "GRPC_PYTHON_BUILD_WITH_CYTHON=1 pip install .") + .format(source) + ) + _ERROR_DIAGNOSES = { errors.CompileError: diagnose_compile_error diff --git a/test/core/security/security_connector_test.c b/test/core/security/security_connector_test.c index 0dcffa40ce413b7a69125a355bb4276e02e20583..ee5435f01ddccf287fdc3beee8dc119216541742 100644 --- a/test/core/security/security_connector_test.c +++ b/test/core/security/security_connector_test.c @@ -1,6 +1,6 @@ /* * - * Copyright 2015, Google Inc. + * Copyright 2015-2016, Google Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -36,6 +36,9 @@ #include "src/core/security/security_connector.h" #include "src/core/security/security_context.h" +#include "src/core/support/env.h" +#include "src/core/support/file.h" +#include "src/core/support/string.h" #include "src/core/tsi/ssl_transport_security.h" #include "src/core/tsi/transport_security.h" #include "test/core/util/test_config.h" @@ -44,6 +47,7 @@ #include <grpc/support/alloc.h> #include <grpc/support/log.h> +#include <grpc/support/string_util.h> #include <grpc/support/useful.h> static int check_transport_security_type(const grpc_auth_context *ctx) { @@ -297,7 +301,66 @@ static void test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context( GRPC_AUTH_CONTEXT_UNREF(ctx, "test"); } -/* TODO(jboeuf): Unit-test tsi_shallow_peer_from_auth_context. */ +static const char *roots_for_override_api = "roots for override api"; + +static grpc_ssl_roots_override_result override_roots_success( + char **pem_root_certs) { + *pem_root_certs = gpr_strdup(roots_for_override_api); + return GRPC_SSL_ROOTS_OVERRIDE_OK; +} + +static grpc_ssl_roots_override_result override_roots_permanent_failure( + char **pem_root_certs) { + return GRPC_SSL_ROOTS_OVERRIDE_FAIL_PERMANENTLY; +} + +static void test_default_ssl_roots(void) { + const char *roots_for_env_var = "roots for env var"; + + char *roots_env_var_file_path; + FILE *roots_env_var_file = + gpr_tmpfile("test_roots_for_env_var", &roots_env_var_file_path); + fwrite(roots_for_env_var, 1, strlen(roots_for_env_var), roots_env_var_file); + fclose(roots_env_var_file); + + /* First let's get the root through the override: set the env to an invalid + value. */ + gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); + grpc_set_ssl_roots_override_callback(override_roots_success); + gpr_slice roots = grpc_get_default_ssl_roots_for_testing(); + char *roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII); + gpr_slice_unref(roots); + GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); + gpr_free(roots_contents); + + /* Now let's set the env var: We should get the contents pointed value + instead. */ + gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, roots_env_var_file_path); + roots = grpc_get_default_ssl_roots_for_testing(); + roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII); + gpr_slice_unref(roots); + GPR_ASSERT(strcmp(roots_contents, roots_for_env_var) == 0); + gpr_free(roots_contents); + + /* Now reset the env var. We should fall back to the value overridden using + the api. */ + gpr_setenv(GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR, ""); + roots = grpc_get_default_ssl_roots_for_testing(); + roots_contents = gpr_dump_slice(roots, GPR_DUMP_ASCII); + gpr_slice_unref(roots); + GPR_ASSERT(strcmp(roots_contents, roots_for_override_api) == 0); + gpr_free(roots_contents); + + /* Now setup a permanent failure for the overridden roots and we should get + an empty slice. */ + grpc_set_ssl_roots_override_callback(override_roots_permanent_failure); + roots = grpc_get_default_ssl_roots_for_testing(); + GPR_ASSERT(GPR_SLICE_IS_EMPTY(roots)); + + /* Cleanup. */ + remove(roots_env_var_file_path); + gpr_free(roots_env_var_file_path); +} int main(int argc, char **argv) { grpc_test_init(argc, argv); @@ -308,6 +371,7 @@ int main(int argc, char **argv) { test_cn_and_one_san_ssl_peer_to_auth_context(); test_cn_and_multiple_sans_ssl_peer_to_auth_context(); test_cn_and_multiple_sans_and_others_ssl_peer_to_auth_context(); + test_default_ssl_roots(); grpc_shutdown(); return 0; diff --git a/test/cpp/end2end/end2end_test.cc b/test/cpp/end2end/end2end_test.cc index 5a414ebc86c114317824906d6babf03d05c4f48c..3ad09aca4cddb8271a555d15d49097f3a3620541 100644 --- a/test/cpp/end2end/end2end_test.cc +++ b/test/cpp/end2end/end2end_test.cc @@ -452,13 +452,18 @@ class End2endTest : public ::testing::TestWithParam<TestScenario> { TestServiceImplDupPkg dup_pkg_service_; }; -static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) { +static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs, + bool with_binary_metadata) { EchoRequest request; EchoResponse response; request.set_message("Hello hello hello hello"); for (int i = 0; i < num_rpcs; ++i) { ClientContext context; + if (with_binary_metadata) { + char bytes[8] = {'\0', '\1', '\2', '\3', '\4', '\5', '\6', (char)i}; + context.AddMetadata("custom-bin", grpc::string(bytes, 8)); + } context.set_compression_algorithm(GRPC_COMPRESS_GZIP); Status s = stub->Echo(&context, request, &response); EXPECT_EQ(response.message(), request.message()); @@ -466,6 +471,30 @@ static void SendRpc(grpc::testing::EchoTestService::Stub* stub, int num_rpcs) { } } +TEST_P(End2endTest, MultipleRpcsWithVariedBinaryMetadataValue) { + ResetStub(); + std::vector<std::thread*> threads; + for (int i = 0; i < 10; ++i) { + threads.push_back(new std::thread(SendRpc, stub_.get(), 10, true)); + } + for (int i = 0; i < 10; ++i) { + threads[i]->join(); + delete threads[i]; + } +} + +TEST_P(End2endTest, MultipleRpcs) { + ResetStub(); + std::vector<std::thread*> threads; + for (int i = 0; i < 10; ++i) { + threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false)); + } + for (int i = 0; i < 10; ++i) { + threads[i]->join(); + delete threads[i]; + } +} + TEST_P(End2endTest, RequestStreamOneRequest) { ResetStub(); EchoRequest request; @@ -803,14 +832,14 @@ class ProxyEnd2endTest : public End2endTest { TEST_P(ProxyEnd2endTest, SimpleRpc) { ResetStub(); - SendRpc(stub_.get(), 1); + SendRpc(stub_.get(), 1, false); } TEST_P(ProxyEnd2endTest, MultipleRpcs) { ResetStub(); std::vector<std::thread*> threads; for (int i = 0; i < 10; ++i) { - threads.push_back(new std::thread(SendRpc, stub_.get(), 10)); + threads.push_back(new std::thread(SendRpc, stub_.get(), 10, false)); } for (int i = 0; i < 10; ++i) { threads[i]->join();