From 1f05f387d65be064d92685b7cfb05d6a9cd5c823 Mon Sep 17 00:00:00 2001 From: Syphax bouazzouni Date: Sat, 18 Nov 2023 18:03:54 +0100 Subject: [PATCH] Feature: Implement CI deployment for development servers (#251) * add test and appliance environment to deploy action * add PRIVATE_CONFIG_REPO environment for the deploy action * revert removing the jumphost step in deploy action * set default deploy branch for each of the environments * set the deployment rails_env to "appliance" for all the environments * use a variable for the config folder path to use in the get_config task * extract deploy ssh_options into the deploy.rb file * add test environment capistrano deploy file * make the jumpbox host and username configurable for the deployment * make PRIVATE_CONFIG_REPO default example use https with github PAT token * remove the on push in development branch auto deploy --- .github/workflows/deploy.yml | 28 ++++++++++++++++++----- config/deploy.rb | 44 +++++++++++++++++++++++++++++++++--- config/deploy/production.rb | 34 +--------------------------- config/deploy/staging.rb | 34 +--------------------------- config/deploy/test.rb | 17 ++++++++++++++ 5 files changed, 82 insertions(+), 75 deletions(-) create mode 100644 config/deploy/test.rb diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b94686e9d7..916bd6cc4c 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -9,6 +9,7 @@ # GH_PAT - github Personal Access Token for accessing private config repo # # SSH_JUMPHOST - ssh jump/proxy host though which deployments have to though if UI nodes live on private network. +# SSH_JUMPHOST_USER - username to use to connect to the ssh jump/proxy. # # DEPLOY_ENC_KEY - key for decrypting deploymnet ssh key residing in config/ # this SSH key is used for accessing jump host, UI nodes, and private github repo. @@ -17,26 +18,27 @@ name: Capistrano Deployment # Controls when the action will run. on: push: - branches: [ development staging ] + branches: [ stage test] # Allows running this workflow manually from the Actions tab workflow_dispatch: inputs: BRANCH: description: 'Branch/tag to deploy' options: - - development - - staging + - stage + - test - master - default: staging + default: test required: true environment: description: 'target environment to deploy to' type: choice options: - - development - staging - production - default: staging + - appliance + - test + default: test jobs: deploy: runs-on: ubuntu-latest @@ -55,6 +57,13 @@ jobs: USER_INPUT_ENVIRONMENT=${{ inputs.environment }} echo "TARGET=${USER_INPUT_ENVIRONMENT:-staging}" >> $GITHUB_ENV + + CONFIG_REPO=${{ secrets.CONFIG_REPO }} + GH_PAT=${{ secrets.GH_PAT }} + echo "PRIVATE_CONFIG_REPO=https://${GH_PAT}@github.com/${CONFIG_REPO}" >> $GITHUB_ENV + + echo "SSH_JUMPHOST=${{ secrets.SSH_JUMPHOST }}" >> $GITHUB_ENV + echo "SSH_JUMPHOST_USER=${{ secrets.SSH_JUMPHOST_USER }}" >> $GITHUB_ENV # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 - uses: ruby/setup-ruby@v1 @@ -69,6 +78,13 @@ jobs: path: deploy_config - name: copy-deployment-config run: cp -r deploy_config/ontoportal_web_ui/${{ inputs.environment }}/* . + # add ssh hostkey so that capistrano doesn't complain + - name: Add jumphost's hostkey to Known Hosts + run: | + mkdir -p ~/.ssh + echo "${{ secrets.SSH_JUMPHOST }}" + ssh-keyscan -H ${{ secrets.SSH_JUMPHOST }} > ~/.ssh/known_hosts + shell: bash - uses: miloserdow/capistrano-deploy@master with: target: ${{ env.TARGET }} # which environment to deploy diff --git a/config/deploy.rb b/config/deploy.rb index 1fe0d99900..f43edb893a 100644 --- a/config/deploy.rb +++ b/config/deploy.rb @@ -11,7 +11,6 @@ # default deployment branch is master which can be overwritten with BRANCH env var # BRANCH env var can be set to specific branch of tag, i.e 'v6.8.1' -set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master' # Default deploy_to directory is /var/www/my_app set :deploy_to, "/srv/ontoportal/#{fetch(:application)}" @@ -42,7 +41,8 @@ set :keep_releases, 5 set :bundle_without, 'development:test' set :bundle_config, { deployment: true } - +set :rails_env, "appliance" +set :config_folder_path, "#{fetch(:application)}/#{fetch(:stage)}" # Defaults to [:web] set :assets_roles, [:web, :app] set :keep_assets, 3 @@ -55,6 +55,43 @@ # If you don't set `:passenger_restart_with_touch`, capistrano-passenger will check what version of passenger you are running # and use `passenger-config restart-app` if it is available in that version. +# you can set custom ssh options +# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options +# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start) +# set it globally +# set :ssh_options, { +# keys: %w(/home/rlisowski/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(password) +# } +# and/or per server +# server 'example.com', +# user: 'user_name', +# roles: %w{web app}, +# ssh_options: { +# user: 'user_name', # overrides user setting above +# keys: %w(/home/user_name/.ssh/id_rsa), +# forward_agent: false, +# auth_methods: %w(publickey password) +# # password: 'please use keys' +# } +# setting per server overrides global ssh_options + +SSH_JUMPHOST = ENV.include?('SSH_JUMPHOST') ? ENV['SSH_JUMPHOST'] : 'jumpbox.hostname.com' +SSH_JUMPHOST_USER = ENV.include?('SSH_JUMPHOST_USER') ? ENV['SSH_JUMPHOST_USER'] : 'username' + +JUMPBOX_PROXY = "#{SSH_JUMPHOST_USER}@#{SSH_JUMPHOST}" +set :ssh_options, { + user: 'ontoportal', + forward_agent: 'true', + keys: %w(config/deploy_id_rsa), + auth_methods: %w(publickey), + # use ssh proxy if UI servers are on a private network + proxy: Net::SSH::Proxy::Command.new("ssh #{JUMPBOX_PROXY} -W %h:%p") +} + +#private git repo for configuraiton +PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'https://your_github_pat_token@github.com/your_organization/ontoportal-configs.git' desc "Check if agent forwarding is working" task :forwarding do on roles(:all) do |h| @@ -83,7 +120,7 @@ TMP_CONFIG_PATH = "/tmp/#{SecureRandom.hex(15)}".freeze on roles(:app) do execute "git clone -q #{PRIVATE_CONFIG_REPO} #{TMP_CONFIG_PATH}" - execute "rsync -a #{TMP_CONFIG_PATH}/#{fetch(:application)}/ #{release_path}/" + execute "rsync -a #{TMP_CONFIG_PATH}/#{fetch(:config_folder_path)}/ #{release_path}/" execute "rm -rf #{TMP_CONFIG_PATH}" end elsif defined?(LOCAL_CONFIG_PATH) @@ -102,6 +139,7 @@ end + after :updating, :get_config after :publishing, :restart after :restart, :clear_cache do diff --git a/config/deploy/production.rb b/config/deploy/production.rb index 78022105df..566c41b81b 100644 --- a/config/deploy/production.rb +++ b/config/deploy/production.rb @@ -6,7 +6,7 @@ # Don't declare `role :all`, it's a meta role role :app, %w{ui1.prd.ontoportal.org ui2.prd.ontoportal.org} role :db, %w{ui1.prd.ontoportal.org} # sufficient to run db:migrate only on one system - +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'master' # Extended Server Syntax # ====================== # This can be used to drop a more detailed server @@ -15,35 +15,3 @@ # extended properties on the server. #server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value set :log_level, :error -# you can set custom ssh options -# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options -# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start) -# set it globally -# set :ssh_options, { -# keys: %w(/home/rlisowski/.ssh/id_rsa), -# forward_agent: false, -# auth_methods: %w(password) -# } -# and/or per server -# server 'example.com', -# user: 'user_name', -# roles: %w{web app}, -# ssh_options: { -# user: 'user_name', # overrides user setting above -# keys: %w(/home/user_name/.ssh/id_rsa), -# forward_agent: false, -# auth_methods: %w(publickey password) -# # password: 'please use keys' -# } -# setting per server overrides global ssh_options -set :ssh_options, { - user: 'deployer', - forward_agent: 'true', - keys: %w(config/deploy_id_rsa), - auth_methods: %w(publickey), - # use ssh proxy if UI servers are on a private network - proxy: Net::SSH::Proxy::Command.new('ssh deployer@sshproxy.ontoportal.org -W %h:%p') -} - -#private git repo for configuraiton -PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'git@github.com:author/private_config_repo.git' diff --git a/config/deploy/staging.rb b/config/deploy/staging.rb index 2ad6dc6cd6..47b158aec2 100644 --- a/config/deploy/staging.rb +++ b/config/deploy/staging.rb @@ -6,7 +6,7 @@ # Don't declare `role :all`, it's a meta role role :app, %w{stageportal.lirmm.fr} role :db, %w{stageportal.lirmm.fr} # sufficient to run db:migrate only on one system - +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'stage' # Extended Server Syntax # ====================== # This can be used to drop a more detailed server @@ -15,35 +15,3 @@ # extended properties on the server. #server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value set :log_level, :error -# you can set custom ssh options -# it's possible to pass any option but you need to keep in mind that net/ssh understand limited list of options -# you can see them in [net/ssh documentation](http://net-ssh.github.io/net-ssh/classes/Net/SSH.html#method-c-start) -# set it globally -# set :ssh_options, { -# keys: %w(/home/rlisowski/.ssh/id_rsa), -# forward_agent: false, -# auth_methods: %w(password) -# } -# and/or per server -# server 'example.com', -# user: 'user_name', -# roles: %w{web app}, -# ssh_options: { -# user: 'user_name', # overrides user setting above -# keys: %w(/home/user_name/.ssh/id_rsa), -# forward_agent: false, -# auth_methods: %w(publickey password) -# # password: 'please use keys' -# } -# setting per server overrides global ssh_options -set :ssh_options, { - user: 'deploy', - forward_agent: 'true', - keys: %w(config/deploy_id_rsa), - auth_methods: %w(publickey), - # use ssh proxy if UI servers are on a private network - # proxy: Net::SSH::Proxy::Command.new('ssh deployer@sshproxy.ontoportal.org -W %h:%p') -} - -#private git repo for configuraiton -PRIVATE_CONFIG_REPO = ENV.include?('PRIVATE_CONFIG_REPO') ? ENV['PRIVATE_CONFIG_REPO'] : 'git@github.com:author/private_config_repo.git' diff --git a/config/deploy/test.rb b/config/deploy/test.rb new file mode 100644 index 0000000000..fcbe1efcec --- /dev/null +++ b/config/deploy/test.rb @@ -0,0 +1,17 @@ +# Simple Role Syntax +# ================== +# Supports bulk-adding hosts to roles, the primary +# server in each group is considered to be the first +# unless any hosts have the primary property set. +# Don't declare `role :all`, it's a meta role +role :app, %w{testportal.lirmm.fr} +role :db, %w{testportal.lirmm.fr} # sufficient to run db:migrate only on one system +# Extended Server Syntax +# ====================== +# This can be used to drop a more detailed server +# definition into the server list. The second argument +# something that quacks like a hash can be used to set +# extended properties on the server. +#server 'example.com', user: 'deploy', roles: %w{web app}, my_property: :my_value +set :log_level, :error +set :branch, ENV.include?('BRANCH') ? ENV['BRANCH'] : 'test'