diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml
index ab41a845..ceb4aeb8 100644
--- a/.github/workflows/backport.yml
+++ b/.github/workflows/backport.yml
@@ -13,6 +13,15 @@ on:
 jobs:
   backport:
     runs-on: ubuntu-latest
+    if: >
+      github.event.pull_request.merged
+      && (
+        github.event.action == 'closed'
+        || (
+          github.event.action == 'labeled'
+          && contains(github.event.label.name, 'backport')
+        )
+      )
     permissions:
       contents: write
       pull-requests: write
@@ -30,6 +39,6 @@ jobs:
         uses: VachaShah/backport@v2.2.0
         with:
           github_token: ${{ steps.github_app_token.outputs.token }}
-          branch_name: backport/backport-${{ github.event.number }}
+          head_template: backport/backport-<%= number %>-to-<%= base %>
           labels_template: "<%= JSON.stringify([...labels, 'autocut']) %>"
           failure_labels: "failed backport"