diff --git a/torchaudio/pipelines/_wav2vec2/impl.py b/torchaudio/pipelines/_wav2vec2/impl.py
index b7bdd7518f..d60fa8adb9 100644
--- a/torchaudio/pipelines/_wav2vec2/impl.py
+++ b/torchaudio/pipelines/_wav2vec2/impl.py
@@ -1350,7 +1350,7 @@ def _get_state_dict(self, dl_kwargs):
         "encoder_ff_interm_features": 4096,
         "encoder_ff_interm_dropout": 0.0,
         "encoder_dropout": 0.1,
-        "encoder_layer_norm_first": False,
+        "encoder_layer_norm_first": True,
         "encoder_layer_drop": 0.05,
         "aux_num_out": None,
     },