Log a warning when the controller clients receive an error (#2499)

The controller client includes a recovery/backoff module that causes
resolutions to be retried when an unexpected error is encountered.
These events are only logged at debugging and trace log levels.

This change updates the destination and policy controller recovery
modules to log unexpected errors as warnings.

linkerd/app/inbound/src/policy/api.rs

@@ -109,7 +109,11 @@ impl Recover<tonic::Status> for GrpcRecover {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected policy controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }

linkerd/app/outbound/src/policy/api.rs

@@ -113,8 +113,12 @@ impl Recover<tonic::Status> for GrpcRecover {
             tonic::Code::InvalidArgument | tonic::Code::FailedPrecondition => Err(status),
             // Indicates no policy for this target
             tonic::Code::NotFound | tonic::Code::Unimplemented => Err(status),
-            _ => {
-                tracing::debug!(%status, "Recovering");
+            code => {
+                tracing::warn!(
+                    grpc.status = %code,
+                    grpc.message = status.message(),
+                    "Unexpected policy controller response; retrying with a backoff",
+                );
                 Ok(self.0.stream())
             }
         }

linkerd/app/src/dst.rs

@@ -94,7 +94,11 @@ impl Recover<tonic::Status> for BackoffUnlessInvalidArgument {
             return Err(status);
         }
 
-        tracing::trace!(%status, "Recovering");
+        tracing::warn!(
+            grpc.status = %status.code(),
+            grpc.message = status.message(),
+            "Unexpected destination controller response; retrying with a backoff",
+        );
         Ok(self.0.stream())
     }
 }