diff --git a/.bashrc b/.bashrc
index 467f2e6..20e1197 100644
--- a/.bashrc
+++ b/.bashrc
@@ -89,7 +89,7 @@ function start_agent {
 }
 
 # Check if we already have an agent running and sourced
-if [ -z "${SSH_AUTH_SOCK}" ]; then
+if [ -z "${SSH_AUTH_SOCK}" -a -s "${PS1}" ]; then
     # Source SSH settings, if applicable
     if [ -f "${SSH_ENV}" ]; then
         . "${SSH_ENV}" > /dev/null