diff --git a/build.json b/build.json
index 6e52d0b6d87e4582cf6c03d2f6da09efbb8b9ab7..9effb7259730a5c4e826349efd8590071f83dbce 100644
--- a/build.json
+++ b/build.json
@@ -1102,6 +1102,7 @@
       "name": "grpc_completion_queue_benchmark",
       "build": "benchmark",
       "language": "c",
+      "run": false,
       "src": [
         "test/core/surface/completion_queue_benchmark.c"
       ],
diff --git a/templates/tools/run_tests/tests.json.template b/templates/tools/run_tests/tests.json.template
index f28a1daad8aea980dba2948d2190a283cb0251e8..bb54489861550794513fabc5e32fd41f1eab39e4 100644
--- a/templates/tools/run_tests/tests.json.template
+++ b/templates/tools/run_tests/tests.json.template
@@ -4,6 +4,6 @@ import json
 
 ${json.dumps([{"name": tgt.name, "language": tgt.language}
               for tgt in targets
-              if tgt.get('run', True)],
+              if tgt.get('run', True) and tgt.build == 'test'],
              sort_keys=True, indent=2)}
 
diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json
index 76a089de87738bfdb1d5dc2d05767e3e042a2bbe..193b164177f7d189d2154f76324985360db54e9c 100644
--- a/tools/run_tests/tests.json
+++ b/tools/run_tests/tests.json
@@ -45,18 +45,10 @@
     "language": "c", 
     "name": "census_statistics_small_log_test"
   }, 
-  {
-    "language": "c", 
-    "name": "census_stats_store_test"
-  }, 
   {
     "language": "c", 
     "name": "census_stub_test"
   }, 
-  {
-    "language": "c", 
-    "name": "census_trace_store_test"
-  }, 
   {
     "language": "c", 
     "name": "census_window_stats_test"
@@ -97,10 +89,6 @@
     "language": "c", 
     "name": "fling_test"
   }, 
-  {
-    "language": "c", 
-    "name": "gen_hpack_tables"
-  }, 
   {
     "language": "c", 
     "name": "gpr_cancellable_test"
@@ -169,10 +157,6 @@
     "language": "c", 
     "name": "grpc_channel_stack_test"
   }, 
-  {
-    "language": "c", 
-    "name": "grpc_completion_queue_benchmark"
-  }, 
   {
     "language": "c", 
     "name": "grpc_completion_queue_test"
@@ -181,10 +165,6 @@
     "language": "c", 
     "name": "grpc_credentials_test"
   }, 
-  {
-    "language": "c", 
-    "name": "grpc_fetch_oauth2"
-  }, 
   {
     "language": "c", 
     "name": "grpc_json_token_test"
@@ -225,10 +205,6 @@
     "language": "c", 
     "name": "lame_client_test"
   }, 
-  {
-    "language": "c", 
-    "name": "low_level_ping_pong_benchmark"
-  }, 
   {
     "language": "c", 
     "name": "message_compress_test"
@@ -309,18 +285,6 @@
     "language": "c++", 
     "name": "end2end_test"
   }, 
-  {
-    "language": "c++", 
-    "name": "grpc_cpp_plugin"
-  }, 
-  {
-    "language": "c++", 
-    "name": "grpc_python_plugin"
-  }, 
-  {
-    "language": "c++", 
-    "name": "grpc_ruby_plugin"
-  }, 
   {
     "language": "c++", 
     "name": "pubsub_publisher_test"