diff --git a/tools/gce_setup/grpc_docker.sh b/tools/gce_setup/grpc_docker.sh
index 2e0265386434f2fb3c590cc9a38dedcdb39b9a1e..1c38582cb8100f184162f39d427c4ff267d3eac9 100755
--- a/tools/gce_setup/grpc_docker.sh
+++ b/tools/gce_setup/grpc_docker.sh
@@ -24,6 +24,10 @@
 # Allows gcloud ssh commands to run on freshly started docker instances.
 _grpc_ensure_gcloud_ssh() {
   local default_key_file="$HOME/.ssh/google_compute_engine"
+  if [ "$HOME" == "/" ]
+  then
+    default_key_file="/root/.ssh/google_compute_engine"
+  fi
   [ -f $default_key_file ] || {
     ssh-keygen -f $default_key_file -N '' > /dev/null || {
       echo "could not precreate $default_key_file" 1>&2