redirector-update.yml

name: Redirector update (cronjob)
on:
  repository_dispatch:
    types: ["Redirector update"]
  schedule:
    - cron: "00 8 * * *"
  workflow_dispatch:

jobs:

  Check:

    name: "Check membership"
    runs-on: Linux
    steps:

      - name: "Check membership"
        uses: armbian/actions/team-check@main
        with:
          ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TEAM: "Release manager"

###################################################################################################

  Test-debs-beta:
    name: "Check debs-beta mirrors"
    needs: Check
    outputs:
      matrix: ${{steps.json.outputs.JSON_CONTENT}}
    runs-on: Linux
    steps:

      - name: Get primary mirrors from database
        id: json
        run: |

          echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
          curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
          "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs-beta&status=active" \
          | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|beta|" | sed "s/\"//g" \
          | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

  Test-debs:
    name: "Check debs mirrors"
    needs: Check
    outputs:
      matrix: ${{steps.json.outputs.JSON_CONTENT}}
    runs-on: Linux
    steps:

      - name: Get primary mirrors from database
        id: json
        run: |

          echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
          curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
          "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs&status=active" \
          | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|apt|" | sed "s/\"//g" \
          | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

  Test-archive:
    name: "Check archive mirrors"
    needs: Check
    outputs:
      matrix: ${{steps.json.outputs.JSON_CONTENT}}
    runs-on: Linux
    steps:

      - name: Get primary mirrors from database
        id: json
        run: |

          echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
          curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
          "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=archive&status=active" \
          | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|archive|" | sed "s/\"//g" \
          | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

  Test-images:
    name: "Check images mirrors"
    needs: Check
    outputs:
      matrix: ${{steps.json.outputs.JSON_CONTENT}}
    runs-on: Linux
    steps:

      - name: Get primary mirrors from database
        id: json
        run: |

          echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
          curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
          "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=images&status=active" \
          | jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|dl|" | sed "s/\"//g" \
          | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
          echo 'EOF' >> $GITHUB_OUTPUT

###################################################################################################

  Ping-debs-beta:
    name: "Check debs-beta"
    runs-on: Linux
    needs: Test-debs-beta
    outputs:
      matrix: ${{needs.Test-debs-beta.outputs.matrix}}
    if: ${{ needs.Test-debs-beta.outputs.matrix != '[]' && needs.Test-debs-beta.outputs.matrix != '' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:

        node: ${{fromJson(needs.Test-debs-beta.outputs.matrix)}}

    steps:

      - name: "Check"
        run: |

          SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
          SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
          ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/beta/control || true)
          CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
          rm -rf downloads/*
          if [[ "$ORIGIN" == "$CHECK" ]]; then
            mkdir -p downloads
            echo $SERVER_URL > downloads/$SERVER_ID
          fi

      - name: Upload JSON
        uses: actions/upload-artifact@v3
        with:
          name: debs-beta
          path: downloads
          if-no-files-found: ignore

  Ping-debs:
    name: "Check debs"
    runs-on: Linux
    needs: Test-debs
    outputs:
      matrix: ${{needs.Test-debs.outputs.matrix}}
    if: ${{ needs.Test-debs.outputs.matrix != '[]' && needs.Test-debs.outputs.matrix != '' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:

        node: ${{fromJson(needs.Test-debs.outputs.matrix)}}

    steps:

      - name: "Check"
        run: |

          SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
          SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
          ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/apt/control || true)
          CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
          rm -rf downloads/*
          if [[ "$ORIGIN" == "$CHECK" ]]; then
            mkdir -p downloads
            echo $SERVER_URL > downloads/$SERVER_ID
          fi

      - name: Upload JSON
        uses: actions/upload-artifact@v3
        with:
          name: debs
          path: downloads
          if-no-files-found: ignore

  Ping-archive:
    name: "Check archive"
    runs-on: Linux
    needs: Test-archive
    outputs:
      matrix: ${{needs.Test-archive.outputs.matrix}}
    if: ${{ needs.Test-archive.outputs.matrix != '[]' && needs.Test-archive.outputs.matrix != '' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:

        node: ${{fromJson(needs.Test-archive.outputs.matrix)}}

    steps:

      - name: "Check"
        run: |

          SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
          SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
          ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/archive/control || true)
          CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
          rm -rf downloads/*
          if [[ "$ORIGIN" == "$CHECK" ]]; then
            mkdir -p downloads
            echo $SERVER_URL > downloads/$SERVER_ID
          fi

      - name: Upload JSON
        uses: actions/upload-artifact@v3
        with:
          name: archive
          path: downloads
          if-no-files-found: ignore

  Ping-images:
    name: "Check images"
    runs-on: Linux
    needs: Test-images
    outputs:
      matrix: ${{needs.Test-images.outputs.matrix}}
    if: ${{ needs.Test-images.outputs.matrix != '[]' && needs.Test-images.outputs.matrix != '' }}
    timeout-minutes: 60
    strategy:
      fail-fast: false
      matrix:

        node: ${{fromJson(needs.Test-images.outputs.matrix)}}

    steps:

      - name: "Check"
        run: |

          SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
          SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
          ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/dl/control || true)
          CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
          rm -rf downloads/*
          if [[ "$ORIGIN" == "$CHECK" ]]; then
            mkdir -p downloads
            echo $SERVER_URL > downloads/$SERVER_ID
          fi

      - name: Upload JSON
        uses: actions/upload-artifact@v3
        with:
          name: images
          path: downloads
          if-no-files-found: ignore

###################################################################################################

  Prepare-debs-beta:
    name: "Generate debs-beta YAML for re-director"
    needs: Ping-debs-beta
    runs-on: Linux
    steps:

      - name: "Clean"
        run: |

          rm -rf downloads/*

      - name: "Download artifacts"
        uses: actions/download-artifact@v3
        with:
          name: debs-beta
          path: downloads

      - name: "Check"
        run: |

          ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
          echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV

      - name: Test debs
        uses: armbian/actions/make-yaml-redirector@main
        with:
          variant: debs-beta
          failoverserver: "${{ env.failoverserver }}"
          port: 8083
          geodb: /scripts/redirect-config/GeoLite2-City.mmdb
          asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
          dl_map: /scripts/redirect-config/all-images.json
          reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
          netbox: ${{ secrets.NETBOX_TOKEN }}

  Prepare-debs:
    name: "Generate debs YAML for re-director"
    needs: Ping-debs
    runs-on: Linux
    steps:

      - name: "Clean"
        run: |

          rm -rf downloads/*

      - name: "Download artifacts"
        uses: actions/download-artifact@v3
        with:
          name: debs
          path: downloads

      - name: "Check"
        run: |

          ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
          echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV

      - name: Test debs
        uses: armbian/actions/make-yaml-redirector@main
        with:
          variant: debs
          failoverserver: "${{ env.failoverserver }}"
          geodb: /scripts/redirect-config/GeoLite2-City.mmdb
          asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
          dl_map: /scripts/redirect-config/all-images.json
          reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
          netbox: ${{ secrets.NETBOX_TOKEN }}

  Prepare-archive:
    name: "Generate archive YAML for re-director"
    needs: Ping-archive
    runs-on: Linux
    steps:

      - name: "Clean"
        run: |

          rm -rf downloads/*

      - name: "Download artifacts"
        uses: actions/download-artifact@v3
        with:
          name: archive
          path: downloads

      - name: "Check"
        run: |

          ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
          echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV

      - name: Test archive
        uses: armbian/actions/make-yaml-redirector@main
        with:
          variant: archive
          failoverserver: "${{ env.failoverserver }}"
          port: 8082
          geodb: /scripts/redirect-config/GeoLite2-City.mmdb
          asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
          dl_map: /scripts/redirect-config/all-images.json
          reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
          netbox: ${{ secrets.NETBOX_TOKEN }}

  Prepare-images:
    name: "Generate images YAML for re-director"
    needs: Ping-images
    runs-on: Linux
    steps:

      - name: "Clean"
        run: |

          rm -rf downloads/*

      - name: "Download artifacts"
        uses: actions/download-artifact@v3
        with:
          name: images
          path: downloads

      - name: "Check"
        run: |

          ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
          echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV

      - name: Test images
        uses: armbian/actions/make-yaml-redirector@main
        with:
          variant: images
          failoverserver: "${{ env.failoverserver }}"
          port: 8081
          geodb: /scripts/redirect-config/GeoLite2-City.mmdb
          asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
          dl_map: /scripts/redirect-config/all-images.json
          reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
          netbox: ${{ secrets.NETBOX_TOKEN }}

###################################################################################################

  Make:
    name: "Upload"
    needs: [ Prepare-debs,Prepare-debs-beta,Prepare-archive,Prepare-images ]
    runs-on: Linux
    steps:

      - name: "Clean"
        run: |

          rm -rf downloads/*

      - uses: robinraju/release-downloader@v1.11
        with:
          repository: "P3TERX/GeoLite.mmdb"
          latest: true
          fileName: "*.mmdb"

      - name: "Download artifacts"
        uses: actions/download-artifact@v3
        with:
          name: redirector-config
          path: downloads

      - name: "Check"
        run: |

          ls -l downloads
          ls -l *.mmdb

      - uses: geekyeggo/delete-artifact@v5
        with:
          name: |
            redirector-config
            debs
            debs-beta
            archive
            images
          failOnError: false

      - name: Install SSH key
        uses: shimataro/ssh-key-action@v2
        with:
          key: ${{ secrets.KEY_UPLOAD }}
          known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
          if_key_exists: replace

      # To do: add runner to redirector, but on clean install. Until then upload files.
      - name: Upload files
        run: |
          rsync -e "ssh -p ${{ secrets.HOST_UPLOAD_PORT }}" -arvc --remove-source-files \
          downloads/dlrouter-archive.yaml \
          downloads/dlrouter-debs-beta.yaml \
          downloads/dlrouter-images.yaml \
          downloads/dlrouter-debs.yaml \
          ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:/incoming/json