Build Docker Images / Push Programs To Server #457
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build Docker Images / Push Programs To Server | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 */16 * * *' | |
| jobs: | |
| push_deb11_to_hub: | |
| name: Push Debian 11 to Docker Hub | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/linux_model_deb_11 | |
| - name: Build and push Debian 11 Docker Image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/midori-ai-pip-deb11-dockerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -f | |
| Build-Subsystem-Programs: | |
| name: Update Subsystem Programs | |
| runs-on: ubuntu-latest | |
| needs: push_deb11_to_hub | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Update Version Name | |
| run: | | |
| mkdir -p /tmp/artifacts | |
| - name: Move everything to working dir | |
| run: | | |
| curl -k --disable --disable-eprt -s https://tea-cup.midori-ai.xyz/download/helper_app.py > helper_app.py | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI-Subsystem-Manager/master/model_installer/yaml_edit.py > yaml_edit.py | |
| curl -k --disable --disable-eprt -s https://raw.githubusercontent.com/lunamidori5/Midori-AI-Subsystem-Manager/master/midori_ai_manager/huggingface_downloader.py > huggingface_downloader.py | |
| - name: Install Dependencies | |
| run: | | |
| pip install pyinstaller | |
| pip install tor-proxy | |
| pip install -r ./Subsystem-Manager/model_installer/requirements.txt | |
| - name: Download Other Programs | |
| run: | | |
| python3 helper_app.py midori_ai_login_app.py | |
| - name: Download the Uploader | |
| run: | | |
| python3 helper_app.py git_uploader.py | |
| - name: Build Programs | |
| run: | | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ helper_app.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ midori_ai_login_app.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ yaml_edit.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ huggingface_downloader.py | |
| pyinstaller --onefile --clean --distpath /tmp/artifacts/ git_uploader.py | |
| - name: Move Exec | |
| run: | | |
| mv /tmp/artifacts/helper_app midori_ai_downloader | |
| mv /tmp/artifacts/midori_ai_login_app midori_ai_login | |
| mv /tmp/artifacts/yaml_edit yaml_edit | |
| mv /tmp/artifacts/huggingface_downloader hf-downloader | |
| mv /tmp/artifacts/git_uploader git_uploader | |
| - name: Upload Raw Python | |
| run: | | |
| chmod +x git_uploader | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "helper_app.py" --filename "helper_app.py" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "yaml_edit.py" --filename "yaml_edit.py" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "huggingface_downloader.py" --filename "huggingface_downloader.py" | |
| - name: Upload Programs | |
| run: | | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "midori_ai_downloader" --filename "midori_ai_downloader" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "yaml_edit" --filename "yaml_edit" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "hf-downloader" --filename "hf-downloader" | |
| ./git_uploader --type Linux --username "${{ secrets.MIDORI_AI_USERNAME }}" --password "${{ secrets.MIDORI_AI_PASSWORD }}" --twoflogin "${{ secrets.MIDORI_AI_TFA }}" --file "midori_ai_login" --filename "midori_ai_login" | |
| push_subsystem_manager_to_hub: | |
| name: Push Subsystem (other os) to Docker Hub | |
| runs-on: ubuntu-latest | |
| needs: Build-Subsystem-Programs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/subsystem_manager | |
| - name: Build and push Subsystem (other os) Docker Image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/midori-ai-subsystem-manager-dockerfile | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -f | |
| push_cpu_to_hub: | |
| name: Push LocalAI CPU to Docker Hub | |
| runs-on: self-hosted | |
| needs: Build-Subsystem-Programs | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_cpu | |
| - name: Make Ver Tags (tags, labels) for Docker | |
| id: sha | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_cpu | |
| tags: | | |
| type=sha,enable=true,priority=100,prefix={{date 'YYYYMMDD-HHmmss' tz='Asia/Tokyo'}}-sha-,suffix=,format=short | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-cpu | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-cpu | |
| push: true | |
| tags: ${{ steps.sha.outputs.tags }} | |
| labels: ${{ steps.sha.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -a -f | |
| docker system prune -a -f | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-gpu | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Build and push CPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-gpu | |
| push: true | |
| tags: ${{ steps.sha.outputs.tags }} | |
| labels: ${{ steps.sha.outputs.labels }} | |
| - name: Clean up Docker Cache | |
| run: | | |
| docker image prune -a -f | |
| docker system prune -a -f | |
| push_gpu_to_hub: | |
| name: Push LocalAI GPUs to Docker Hub | |
| strategy: | |
| matrix: | |
| gpu_type: [intelf16, intelf32, hipblas, nvidia] | |
| needs: push_cpu_to_hub | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: recursive | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@master | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| logout: false | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_${{ matrix.gpu_type }}_gpu | |
| - name: Make Ver Tags (tags, labels) for Docker | |
| id: sha | |
| uses: docker/metadata-action@master | |
| with: | |
| images: lunamidori5/midori_ai_subsystem_localai_${{ matrix.gpu_type }}_gpu | |
| tags: | | |
| type=sha,enable=true,priority=100,prefix={{date 'YYYYMMDD-HHmmss' tz='Asia/Tokyo'}}-sha-,suffix=,format=short | |
| - name: Build and push GPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-${{ matrix.gpu_type }}-gpu | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - name: Build and push GPU LocalAI Docker image | |
| uses: docker/build-push-action@master | |
| with: | |
| context: ./Subsystem-Manager/subsystem_docker_stuff/localai/ | |
| file: ./Subsystem-Manager/subsystem_docker_stuff/localai/localai-subsystem-dockerfile-${{ matrix.gpu_type }}-gpu | |
| push: true | |
| tags: ${{ steps.sha.outputs.tags }} | |
| labels: ${{ steps.sha.outputs.labels }} | |
| clean_docker_mess: | |
| name: Clean up Docker Cache | |
| strategy: | |
| matrix: | |
| gpu_type: [1, 2, 3] | |
| for_each: [1, 2, 3] | |
| needs: push_gpu_to_hub | |
| runs-on: self-hosted | |
| steps: | |
| - name: Clean up Docker Cache | |
| run: | | |
| nohup docker image prune -a -f & | |
| nohup docker system prune -a -f & |