diff --git a/.github/scripts/validate_poetry.sh b/.github/scripts/validate_poetry.sh
index c4e4fd154..3c41d5b45 100644
--- a/.github/scripts/validate_poetry.sh
+++ b/.github/scripts/validate_poetry.sh
@@ -26,10 +26,11 @@ if [[ ${MATRIX_CHANNEL} != "release" ]]; then
     fi
 else
     export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
+    poetry source add --priority=explicit pytorch "https://download.pytorch.org/whl/${MATRIX_DESIRED_CUDA}"
     if [[ ${TORCH_ONLY} == 'true' ]]; then
         poetry --quiet add torch
     else
-        poetry --quiet add torch torchaudio torchvision
+        poetry --quiet add --source pytorch torch torchaudio torchvision
     fi
 fi
 
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
index 12335e843..3c5aac2eb 100644
--- a/.github/workflows/validate-linux-binaries.yml
+++ b/.github/workflows/validate-linux-binaries.yml
@@ -68,11 +68,8 @@ jobs:
         eval "$(conda shell.bash hook)"
 
         # Special case PyPi installation package. And Install of PyPi package via poetry
-        if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" ]] && \
-           ([[ ${MATRIX_GPU_ARCH_VERSION} == "12.1" && ${MATRIX_CHANNEL} != "release" ]] || \
-            [[ ${MATRIX_GPU_ARCH_VERSION} == "11.7" && ${MATRIX_CHANNEL} == "release" ]]); then
+        if [[ ${MATRIX_PACKAGE_TYPE} == "manywheel" && ${MATRIX_GPU_ARCH_VERSION} == "12.1" ]]; then
           source ./.github/scripts/validate_pipy.sh
-          # temporary disable poetry check
           source ./.github/scripts/validate_poetry.sh
         fi