diff --git a/test/cpp/qps/async_streaming_ping_pong_test.cc b/test/cpp/qps/async_streaming_ping_pong_test.cc index a1822b7e15623bbc4f515eeeef7956b02cb2a6fb..3106f26d66fb185148af602835e75fed786fdf6b 100644 --- a/test/cpp/qps/async_streaming_ping_pong_test.cc +++ b/test/cpp/qps/async_streaming_ping_pong_test.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <grpc/support/log.h> #include <signal.h> @@ -47,6 +49,9 @@ static const int BENCHMARK = 10; static void RunAsyncStreamingPingPong() { gpr_log(GPR_INFO, "Running Async Streaming Ping Pong"); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + ClientConfig client_config; client_config.set_client_type(ASYNC_CLIENT); client_config.set_enable_ssl(false); @@ -64,8 +69,10 @@ static void RunAsyncStreamingPingPong() { const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); - ReportQPS(result); - ReportLatency(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_QPS); + types.insert(grpc::testing::ReportType::REPORT_LATENCY); + reporters_registry.Report({client_config, server_config, result}, types); } } // namespace testing diff --git a/test/cpp/qps/async_unary_ping_pong_test.cc b/test/cpp/qps/async_unary_ping_pong_test.cc index 8b037a865628b1f3f3f942f5d29579d5607ed34c..a51badd9c1ed9e5c6afd4a0671263bd3e1a329b7 100644 --- a/test/cpp/qps/async_unary_ping_pong_test.cc +++ b/test/cpp/qps/async_unary_ping_pong_test.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <grpc/support/log.h> #include <signal.h> @@ -47,6 +49,9 @@ static const int BENCHMARK = 10; static void RunAsyncUnaryPingPong() { gpr_log(GPR_INFO, "Running Async Unary Ping Pong"); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + ClientConfig client_config; client_config.set_client_type(ASYNC_CLIENT); client_config.set_enable_ssl(false); @@ -64,8 +69,10 @@ static void RunAsyncUnaryPingPong() { const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); - ReportQPS(result); - ReportLatency(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_QPS); + types.insert(grpc::testing::ReportType::REPORT_LATENCY); + reporters_registry.Report({client_config, server_config, result}, types); } } // namespace testing diff --git a/test/cpp/qps/qps_driver.cc b/test/cpp/qps/qps_driver.cc index fc8e04201cd0f62e55e4009d33742cdeda60c0c0..3aa215b44856676de4730fc64189c71079f335b3 100644 --- a/test/cpp/qps/qps_driver.cc +++ b/test/cpp/qps/qps_driver.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <gflags/gflags.h> #include <grpc/support/log.h> @@ -67,10 +69,17 @@ using grpc::testing::ClientType; using grpc::testing::ServerType; using grpc::testing::RpcType; using grpc::testing::ResourceUsage; +using grpc::testing::ReportersRegistry; +using grpc::testing::GprLogReporter; +using grpc::testing::ReportData; +using grpc::testing::ReportType; int main(int argc, char** argv) { grpc::testing::InitTest(&argc, &argv, true); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + RpcType rpc_type; GPR_ASSERT(RpcType_Parse(FLAGS_rpc_type, &rpc_type)); @@ -103,14 +112,13 @@ int main(int argc, char** argv) { FLAGS_server_threads < FLAGS_client_channels * FLAGS_outstanding_rpcs_per_channel)); - auto result = RunScenario(client_config, FLAGS_num_clients, - server_config, FLAGS_num_servers, - FLAGS_warmup_seconds, FLAGS_benchmark_seconds, - FLAGS_local_workers); + const auto result = RunScenario( + client_config, FLAGS_num_clients, server_config, FLAGS_num_servers, + FLAGS_warmup_seconds, FLAGS_benchmark_seconds, FLAGS_local_workers); - ReportQPSPerCore(result, server_config); - ReportLatency(result); - ReportTimes(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_ALL); + reporters_registry.Report({client_config, server_config, result}, types); return 0; } diff --git a/test/cpp/qps/qps_test.cc b/test/cpp/qps/qps_test.cc index f567e4cf06104960c39c30d92cc089168c2bdf02..2f72519397d963339cdcd3ae2bd8773230875be0 100644 --- a/test/cpp/qps/qps_test.cc +++ b/test/cpp/qps/qps_test.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <grpc/support/log.h> #include <signal.h> @@ -47,6 +49,9 @@ static const int BENCHMARK = 10; static void RunQPS() { gpr_log(GPR_INFO, "Running QPS test"); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + ClientConfig client_config; client_config.set_client_type(ASYNC_CLIENT); client_config.set_enable_ssl(false); @@ -64,8 +69,11 @@ static void RunQPS() { const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); - ReportQPSPerCore(result, server_config); - ReportLatency(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_QPS_PER_CORE); + types.insert(grpc::testing::ReportType::REPORT_LATENCY); + reporters_registry.Report({client_config, server_config, result}, types); + } } // namespace testing diff --git a/test/cpp/qps/report.cc b/test/cpp/qps/report.cc index 29d88da344a23f0b6492fdba9f8e0183ef755a74..72b3a3643d55ee21f854ceffadf5ee5c88d9a2f0 100644 --- a/test/cpp/qps/report.cc +++ b/test/cpp/qps/report.cc @@ -39,27 +39,73 @@ namespace grpc { namespace testing { -// QPS: XXX -void ReportQPS(const ScenarioResult& result) { +// ReporterRegistry implementation. +void ReportersRegistry::Register(const Reporter* reporter) { + reporters_.emplace_back(reporter); +} + +std::vector<string> ReportersRegistry::GetNamesRegistered() const { + std::vector<string> names; + for (const auto& reporter : reporters_) { + names.push_back(reporter->name()); + } + return names; +} + +void ReportersRegistry::Report(const ReportData& data, + const std::set<ReportType>& types) const { + for (const auto& reporter : reporters_) { + reporter->Report(data, types); + } +} + +// Reporter implementation. +void Reporter::Report(const ReportData& data, + const std::set<ReportType>& types) const { + for (ReportType rtype : types) { + bool all = false; + switch (rtype) { + case REPORT_ALL: + all = true; + case REPORT_QPS: + ReportQPS(data.scenario_result); + if (!all) break; + case REPORT_QPS_PER_CORE: + ReportQPSPerCore(data.scenario_result, data.server_config); + if (!all) break; + case REPORT_LATENCY: + ReportLatency(data.scenario_result); + if (!all) break; + case REPORT_TIMES: + ReportTimes(data.scenario_result); + if (!all) break; + } + if (all) break; + } +} + +// GprLogReporter implementation. +void GprLogReporter::ReportQPS(const ScenarioResult& result) const { gpr_log(GPR_INFO, "QPS: %.1f", result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; })); } -// QPS: XXX (YYY/server core) -void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& server_config) { - auto qps = +void GprLogReporter::ReportQPSPerCore(const ScenarioResult& result, + const ServerConfig& server_config) const { + auto qps = result.latencies.Count() / average(result.client_resources, [](ResourceUsage u) { return u.wall_time; }); - gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps, qps/server_config.threads()); + gpr_log(GPR_INFO, "QPS: %.1f (%.1f/server core)", qps, + qps / server_config.threads()); } -// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us -void ReportLatency(const ScenarioResult& result) { - gpr_log(GPR_INFO, "Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us", +void GprLogReporter::ReportLatency(const ScenarioResult& result) const { + gpr_log(GPR_INFO, + "Latencies (50/90/95/99/99.9%%-ile): %.1f/%.1f/%.1f/%.1f/%.1f us", result.latencies.Percentile(50) / 1000, result.latencies.Percentile(90) / 1000, result.latencies.Percentile(95) / 1000, @@ -67,7 +113,7 @@ void ReportLatency(const ScenarioResult& result) { result.latencies.Percentile(99.9) / 1000); } -void ReportTimes(const ScenarioResult& result) { +void GprLogReporter::ReportTimes(const ScenarioResult& result) const { gpr_log(GPR_INFO, "Server system time: %.2f%%", 100.0 * sum(result.server_resources, [](ResourceUsage u) { return u.system_time; }) / diff --git a/test/cpp/qps/report.h b/test/cpp/qps/report.h index 343e426ca4ac74ecd877d7d5010933ac9550bde9..e62b9deb32be55cd092982f9fa21898c309824de 100644 --- a/test/cpp/qps/report.h +++ b/test/cpp/qps/report.h @@ -34,22 +34,112 @@ #ifndef TEST_QPS_REPORT_H #define TEST_QPS_REPORT_H +#include <memory> +#include <set> +#include <vector> +#include <grpc++/config.h> + #include "test/cpp/qps/driver.h" +#include "test/cpp/qps/qpstest.grpc.pb.h" namespace grpc { namespace testing { -// QPS: XXX -void ReportQPS(const ScenarioResult& result); -// QPS: XXX (YYY/server core) -void ReportQPSPerCore(const ScenarioResult& result, const ServerConfig& config); -// Latency (50/90/95/99/99.9%-ile): AA/BB/CC/DD/EE us -void ReportLatency(const ScenarioResult& result); -// Server system time: XX% -// Server user time: XX% -// Client system time: XX% -// Client user time: XX% -void ReportTimes(const ScenarioResult& result); +/** General set of data required for report generation. */ +struct ReportData { + const ClientConfig& client_config; + const ServerConfig& server_config; + const ScenarioResult& scenario_result; +}; + +/** Specifies the type of performance report we are interested in. + * + * \note The special type \c REPORT_ALL is equivalent to specifying all the + * other fields. */ +enum ReportType { + /** Equivalent to the combination of all other fields. */ + REPORT_ALL, + /** Report only QPS information. */ + REPORT_QPS, + /** Report only QPS per core information. */ + REPORT_QPS_PER_CORE, + /** Report latency info for the 50, 90, 95, 99 and 99.9th percentiles. */ + REPORT_LATENCY, + /** Report user and system time. */ + REPORT_TIMES +}; + +class Reporter; + +/** A registry of Reporter instances. + * + * Instances registered will be taken into account by the Report() method. + */ +class ReportersRegistry { + public: + /** Adds the \c reporter to the registry. + * \attention Takes ownership of \c reporter. */ + void Register(const Reporter* reporter); + + /** Returns the names of the registered \c Reporter instances. */ + std::vector<string> GetNamesRegistered() const; + + /** Triggers the reporting for all registered \c Reporter instances. + * + * \param data Configuration and results for the scenario being reported. + * \param types A collection of report types to include in the report. */ + void Report(const ReportData& data, + const std::set<ReportType>& types) const; + + private: + std::vector<std::unique_ptr<const Reporter> > reporters_; +}; + +/** Interface for all reporters. */ +class Reporter { + public: + /** Construct a reporter with the given \a name. */ + Reporter(const string& name) : name_(name) {} + + /** Returns this reporter's name. + * + * Names are constants, set at construction time. */ + string name() const { return name_; } + + /** Template method responsible for the generation of the requested types. */ + void Report(const ReportData& data, const std::set<ReportType>& types) const; + + protected: + /** Reports QPS for the given \a result. */ + virtual void ReportQPS(const ScenarioResult& result) const = 0; + /** Reports QPS per core as (YYY/server core). */ + virtual void ReportQPSPerCore(const ScenarioResult& result, + const ServerConfig& config) const = 0; + /** Reports latencies for the 50, 90, 95, 99 and 99.9 percentiles, in ms. */ + virtual void ReportLatency(const ScenarioResult& result) const = 0; + + /** Reports system and user time for client and server systems. */ + virtual void ReportTimes(const ScenarioResult& result) const = 0; + + private: + const string name_; +}; + + +// Reporters. + +/** Reporter to gpr_log(GPR_INFO). */ +class GprLogReporter : public Reporter { + public: + GprLogReporter(const string& name) : Reporter(name) {} + + private: + void ReportQPS(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportQPSPerCore(const ScenarioResult& result, + const ServerConfig& config) const GRPC_OVERRIDE; + void ReportLatency(const ScenarioResult& result) const GRPC_OVERRIDE; + void ReportTimes(const ScenarioResult& result) const GRPC_OVERRIDE; +}; } // namespace testing } // namespace grpc diff --git a/test/cpp/qps/sync_streaming_ping_pong_test.cc b/test/cpp/qps/sync_streaming_ping_pong_test.cc index 48c7ff63e03bd1896dd691dcdd2e247947c2e5a5..ddc1573bfdd0aed6a845f69ebe89a3558fafbefb 100644 --- a/test/cpp/qps/sync_streaming_ping_pong_test.cc +++ b/test/cpp/qps/sync_streaming_ping_pong_test.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <grpc/support/log.h> #include <signal.h> @@ -47,6 +49,9 @@ static const int BENCHMARK = 10; static void RunSynchronousStreamingPingPong() { gpr_log(GPR_INFO, "Running Synchronous Streaming Ping Pong"); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + ClientConfig client_config; client_config.set_client_type(SYNCHRONOUS_CLIENT); client_config.set_enable_ssl(false); @@ -63,8 +68,11 @@ static void RunSynchronousStreamingPingPong() { const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); - ReportQPS(result); - ReportLatency(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_QPS); + types.insert(grpc::testing::ReportType::REPORT_LATENCY); + reporters_registry.Report({client_config, server_config, result}, types); + } } // namespace testing diff --git a/test/cpp/qps/sync_unary_ping_pong_test.cc b/test/cpp/qps/sync_unary_ping_pong_test.cc index 4c4de6377b8f22de244c707cf139983a86a30739..a1bd1f1705db9d935b6b0571918ede16ed0295d2 100644 --- a/test/cpp/qps/sync_unary_ping_pong_test.cc +++ b/test/cpp/qps/sync_unary_ping_pong_test.cc @@ -31,6 +31,8 @@ * */ +#include <set> + #include <grpc/support/log.h> #include <signal.h> @@ -47,6 +49,9 @@ static const int BENCHMARK = 10; static void RunSynchronousUnaryPingPong() { gpr_log(GPR_INFO, "Running Synchronous Unary Ping Pong"); + ReportersRegistry reporters_registry; + reporters_registry.Register(new GprLogReporter("LogReporter")); + ClientConfig client_config; client_config.set_client_type(SYNCHRONOUS_CLIENT); client_config.set_enable_ssl(false); @@ -63,8 +68,10 @@ static void RunSynchronousUnaryPingPong() { const auto result = RunScenario(client_config, 1, server_config, 1, WARMUP, BENCHMARK, -2); - ReportQPS(result); - ReportLatency(result); + std::set<ReportType> types; + types.insert(grpc::testing::ReportType::REPORT_QPS); + types.insert(grpc::testing::ReportType::REPORT_LATENCY); + reporters_registry.Report({client_config, server_config, result}, types); } } // namespace testing