repo_search.yml

name: SearchRepos

# Controls when the workflow will run
on:
  schedule:
    # * is a special character in YAML so you have to quote this string
    - cron:  '26 4/12 * * *'
  
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  search:
    name: Search
    runs-on: ubuntu-latest

    strategy:
      max-parallel: 10
      fail-fast: false
      matrix:
       search_str: 
         # comments indicate a snapshot of how many results there were 
         # for a given search around the time it was added to the list
         # 106
         - vuln poc
         # 110
         - heap overflow
         # 119
         - cnvd-c OR cnvd-2 OR cnnvd-2
         # 126
         - zeroday
         # 191
         - rce poc
         # 219
         - attack poc
         # 2652
         - metasploit module OR payload
         # 279
         - command injection
         # 383
         - 0day
         # 441 
         - vulnerability poc
         # 526
         - remote code execution
         # 960
         - sploit
         # 1114
         - cve poc
         # 2021
         - shellcode
         # 2267
         - rce 
         # 4626
         - cve-2 OR cve_2
         # 18176
         - exploit
         # 17
         - UEFI bootkit
         # 313
         - uac bypass
         # 12
         - secure boot bypass
         # 35
         - exploitation persistence
#          # 2944
#          - '"privilege escalation" OR privesc'
#          # 1504
#          - buffer overflow
#          # 4383 (noisy)
#          - zero-day
#          # 21863 (very noisy)
#          - 0-day
#          # 502 (noisy)
#          - 1day
        
    steps:
      - name: Get today and yesterday dates
        id: date
        run: |
          echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
          echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT
    
      - name: checkout
        uses: actions/checkout@v4
      
      - name: setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'
      
      - name: setup env
        uses: ./.github/actions/setup_my_env
          
      - name: search github
        uses: ./.github/actions/single_search
        with:
          start_date: ${{ steps.date.outputs.yesterday }}
          end_date: ${{ steps.date.outputs.today }}
          search_str: ${{ matrix.search_str }}
          token: ${{ secrets.GITHUB_TOKEN }}          

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          commit_msg: Search for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} ${{ matrix.search_str }}

      - name: push results
        uses: ./.github/actions/pull_rebase_push_retry

  update_summaries:
    name: Update Summaries
    needs: search
    runs-on: ubuntu-latest

    steps:
      - name: Get dates
        id: date
        run: |
          echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
          echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT
          echo "this_month=$(date +'%Y-%m')" >> $GITHUB_OUTPUT
          echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT

      - name: checkout
        uses: actions/checkout@v4
        with:
          token: ${{ secrets.VUL_DRILLER_TOKEN }}

      - name: setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: setup env
        uses: ./.github/actions/setup_my_env

      - name: git pull
        run: git pull

      - name: summarize yesterday
        uses: ./.github/actions/generate_summaries
        with:
          ymd_option: '--day'
          ymd_value: ${{ steps.date.outputs.yesterday }}

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          commit_msg: Update Summaries for day ${{ steps.date.outputs.yesterday }}

      - name: summarize today
        uses: ./.github/actions/generate_summaries
        with:
          ymd_option: '--day'
          ymd_value: ${{ steps.date.outputs.today }}

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          commit_msg: Update Summaries for day ${{ steps.date.outputs.today }}

      - name: summarize this month
        uses: ./.github/actions/generate_summaries
        with:
          ymd_option: '--month'
          ymd_value: ${{ steps.date.outputs.this_month }}

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          commit_msg: Update Summaries for month ${{ steps.date.outputs.this_month }}

      - name: push results
        uses: ./.github/actions/pull_rebase_push_retry

  deep_dive:
    name: Deep Dive
    needs: update_summaries
    runs-on: ubuntu-latest

    strategy:
      max-parallel: 11
      fail-fast: false
      matrix:
        mod: [ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9 ]

    steps:
      - name: Get dates
        id: date
        run: |
          echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
          echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT
          echo "last_day=$(date -d yesterday +'%d')" >> $GITHUB_OUTPUT
          echo "this_day=$(date +'%d')" >> $GITHUB_OUTPUT
          echo "this_month=$(date +'%m')" >> $GITHUB_OUTPUT
          echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT

      - name: checkout
        uses: actions/checkout@v4

      - name: setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: setup env
        uses: ./.github/actions/setup_my_env

      # in case things have changed since we spawned
      - name: git pull
        run: git pull

      - name: do_deep_dive
        uses: ./.github/actions/deep_dive
        with:
          in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.last_day }}
          mod: ${{ matrix.mod }}
          divisor: ${{ 10 }}
          token: ${{ secrets.VUL_DRILLER_TOKEN }}
          max_age: 7200

      - name: do_deep_dive
        uses: ./.github/actions/deep_dive
        with:
          in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.this_day }}
          mod: ${{ matrix.mod }}
          divisor: ${{ 10 }}
          token: ${{ secrets.VUL_DRILLER_TOKEN }}
          max_age: 3600

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          results_dir: data
          commit_msg: Deep Dive for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} chunk ${{ matrix.mod }}/10

      - name: push result
        uses: ./.github/actions/pull_rebase_push_retry

  repo2vulid:
    name: Repo to Vul IDs
    needs: deep_dive
    runs-on: ubuntu-latest

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: setup python
        uses: actions/setup-python@v5
        with:
          python-version: '3.9'

      - name: setup env
        uses: ./.github/actions/setup_my_env

      # in case things have changed since we spawned
      - name: git pull
        run: git pull

      - name: extract vul ids
        run: repo_to_vul_id --verbose --input_dir data/repo_id

      - name: add and commit
        uses: ./.github/actions/commit_results
        with:
          results_dir: data/vul_id
          commit_msg: Update vulnerability ID data

      - name: push result
        uses: ./.github/actions/pull_rebase_push_retry