diff --git a/examples/cli/cli.cpp b/examples/cli/cli.cpp
index e505bf0e1..c64976a56 100644
--- a/examples/cli/cli.cpp
+++ b/examples/cli/cli.cpp
@@ -465,7 +465,15 @@ static void output_txt(struct whisper_context * ctx, std::ofstream & fout, const
             speaker = estimate_diarization_speaker(pcmf32s, t0, t1);
         }
 
-        fout << speaker << text << "\n";
+        if (!speaker.empty()) {
+            fout << speaker << text;
+        } else {
+            while (*text == ' ' || *text == '\t') {
+                text++;
+            }
+            fout << text;
+        }
+        fout << "\n";
     }
 }