From 967a89360cf53eb19eb67a61acc0b7fdee85e5fc Mon Sep 17 00:00:00 2001 From: Mokhtar Naamani Date: Sat, 24 Aug 2024 13:29:03 +0400 Subject: [PATCH] docker compose instead of docker-compose --- docker-compose-no-bind-volumes.yml | 7 ++++++- docker-compose.elasticsearch.yml | 2 -- docker-compose.localstack.yml | 1 - docker-compose.storage-squid.yml | 2 -- docker-compose.yml | 5 ++--- query-node/kill.sh | 12 +++++------ query-node/reset-processor.sh | 4 ++-- query-node/start.sh | 10 +++++----- scripts/save-to-docker-images.sh | 6 +++--- start-elasticsearch-stack.sh | 14 ++++++------- start-multistorage.sh | 20 +++++++++---------- start-orion.sh | 16 +++++++-------- start.sh | 16 +++++++-------- tests/network-tests/run-node-docker.sh | 2 +- .../run-runtime-upgrade-tests.sh | 4 ++-- tests/network-tests/run-tests.sh | 6 +++--- tests/network-tests/start-storage.sh | 12 +++++------ tests/network-tests/test-setup-new-chain.sh | 2 +- 18 files changed, 70 insertions(+), 71 deletions(-) diff --git a/docker-compose-no-bind-volumes.yml b/docker-compose-no-bind-volumes.yml index 4aab4d0180..bc0279c7d3 100644 --- a/docker-compose-no-bind-volumes.yml +++ b/docker-compose-no-bind-volumes.yml @@ -1,5 +1,4 @@ # Complete joystream development network -version: '3.4' services: joystream-node: image: joystream/node:$JOYSTREAM_NODE_TAG @@ -37,6 +36,9 @@ services: - ACCOUNT_URI=${COLOSSUS_1_TRANSACTOR_URI} # - OTEL_EXPORTER_OTLP_ENDPOINT=http://apm-server:8200 # - OTEL_RESOURCE_ATTRIBUTES=service.name=colossus-1,deployment.environment=production + - AWS_BUCKET_NAME=test-bucket-1 + - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT} + - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED} entrypoint: ['yarn'] command: [ 'start', '--worker=${COLOSSUS_1_WORKER_ID}', '--port=3333', '--uploads=/data/uploads', @@ -106,6 +108,9 @@ services: environment: # ACCOUNT_URI overrides command line arg --accountUri - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI} + - AWS_BUCKET_NAME=test-bucket-2 + - LOCALSTACK_ENDPOINT=${LOCALSTACK_ENDPOINT} + - LOCALSTACK_ENABLED=${LOCALSTACK_ENABLED} entrypoint: ['yarn', 'storage-node'] command: [ 'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads', diff --git a/docker-compose.elasticsearch.yml b/docker-compose.elasticsearch.yml index 5e2a6ea762..0bb114b623 100644 --- a/docker-compose.elasticsearch.yml +++ b/docker-compose.elasticsearch.yml @@ -1,5 +1,3 @@ -version: '3.4' - services: # Ref: https://www.elastic.co/guide/en/elasticsearch/reference/8.7/docker.html elasticsearch: diff --git a/docker-compose.localstack.yml b/docker-compose.localstack.yml index f17af893a0..be3f09d163 100644 --- a/docker-compose.localstack.yml +++ b/docker-compose.localstack.yml @@ -1,4 +1,3 @@ -version: '3.4' services: localstack: container_name: local_stack diff --git a/docker-compose.storage-squid.yml b/docker-compose.storage-squid.yml index 5633367cf7..771eb74c45 100644 --- a/docker-compose.storage-squid.yml +++ b/docker-compose.storage-squid.yml @@ -1,5 +1,3 @@ -version: '3' - services: squid_db: container_name: squid_db diff --git a/docker-compose.yml b/docker-compose.yml index 582718fb1b..1b667b8911 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,4 @@ # Complete joystream development network -version: '3.4' services: joystream-node: image: joystream/node:$JOYSTREAM_NODE_TAG @@ -256,7 +255,7 @@ services: " indexer: - image: joystream/hydra-indexer:v5.0.0-alpha.8 + image: hydra-indexer container_name: indexer restart: unless-stopped env_file: @@ -278,7 +277,7 @@ services: sh -c "yarn db:bootstrap && yarn start:prod" hydra-indexer-gateway: - image: joystream/hydra-indexer-gateway:5.0.0-alpha.8 + image: hydra-indexer-gateway container_name: hydra-indexer-gateway restart: unless-stopped env_file: diff --git a/query-node/kill.sh b/query-node/kill.sh index 0f335c96eb..15a8b02cb1 100755 --- a/query-node/kill.sh +++ b/query-node/kill.sh @@ -5,10 +5,10 @@ SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" cd $SCRIPT_PATH # Only remove query-node related services -docker-compose -f ../docker-compose.yml rm -vsf processor -docker-compose -f ../docker-compose.yml rm -vsf graphql-server -docker-compose -f ../docker-compose.yml rm -vsf indexer -docker-compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway -docker-compose -f ../docker-compose.yml rm -vsf redis -docker-compose -f ../docker-compose.yml rm -vsf db +docker compose -f ../docker-compose.yml rm -vsf processor +docker compose -f ../docker-compose.yml rm -vsf graphql-server +docker compose -f ../docker-compose.yml rm -vsf indexer +docker compose -f ../docker-compose.yml rm -vsf hydra-indexer-gateway +docker compose -f ../docker-compose.yml rm -vsf redis +docker compose -f ../docker-compose.yml rm -vsf db docker volume rm joystream_query-node-data diff --git a/query-node/reset-processor.sh b/query-node/reset-processor.sh index 028e9c6dff..22e47acb72 100755 --- a/query-node/reset-processor.sh +++ b/query-node/reset-processor.sh @@ -4,7 +4,7 @@ set -e SCRIPT_PATH="$(dirname "${BASH_SOURCE[0]}")" cd $SCRIPT_PATH -docker-compose -f ../docker-compose.yml rm -vsf processor -docker-compose -f ../docker-compose.yml rm -vsf graphql-server +docker compose -f ../docker-compose.yml rm -vsf processor +docker compose -f ../docker-compose.yml rm -vsf graphql-server docker exec db psql -U postgres -c "DROP DATABASE query_node_processor;" ./start.sh \ No newline at end of file diff --git a/query-node/start.sh b/query-node/start.sh index e9bad75048..69e3630888 100755 --- a/query-node/start.sh +++ b/query-node/start.sh @@ -8,16 +8,16 @@ cd $SCRIPT_PATH [ ! -d "generated/" ] && yarn build # Bring up db -docker-compose -f ../docker-compose.yml up -d db +docker compose -f ../docker-compose.yml up -d db echo "Waiting for the db to be ready..." sleep 5 # Start indexer and gateway -docker-compose -f ../docker-compose.yml up -d indexer -docker-compose -f ../docker-compose.yml up -d hydra-indexer-gateway +docker compose -f ../docker-compose.yml up -d indexer +docker compose -f ../docker-compose.yml up -d hydra-indexer-gateway # Start processor -docker-compose -f ../docker-compose.yml up -d processor +docker compose -f ../docker-compose.yml up -d processor echo "Waiting for processor to be ready..." && sleep 30 if [[ "$OSTYPE" == "darwin"* ]]; then # On Docker Desktop things take a bit longer to startup @@ -25,5 +25,5 @@ if [[ "$OSTYPE" == "darwin"* ]]; then fi # Start graphql-server -docker-compose -f ../docker-compose.yml up -d graphql-server +docker compose -f ../docker-compose.yml up -d graphql-server echo "Waiting for graphql-server to be ready..." && sleep 30 diff --git a/scripts/save-to-docker-images.sh b/scripts/save-to-docker-images.sh index 7c2914ccbb..2b828b056d 100755 --- a/scripts/save-to-docker-images.sh +++ b/scripts/save-to-docker-images.sh @@ -2,10 +2,10 @@ set -e # clean start -docker-compose -f ../docker-compose.yml down -v +docker compose -f ../docker-compose.yml down -v function cleanup() { - docker-compose -f ../docker-compose.yml down -v + docker compose -f ../docker-compose.yml down -v } trap cleanup EXIT @@ -21,7 +21,7 @@ if [[ -z $JOYSTREAM_NODE_TAG ]]; then fi # start node image, network and volume -docker-compose -f ../docker-compose.yml up -d joystream-node +docker compose -f ../docker-compose.yml up -d joystream-node # copy native runtime docker cp ../target/release/joystream-node joystream-node:/joystream/node diff --git a/start-elasticsearch-stack.sh b/start-elasticsearch-stack.sh index 909031b182..6a47ac348b 100755 --- a/start-elasticsearch-stack.sh +++ b/start-elasticsearch-stack.sh @@ -15,10 +15,10 @@ ELASTIC_USERNAME=${ELASTIC_USERNAME:="elastic"} ELASTIC_PASSWORD=${ELASTIC_PASSWORD:="password"} # Remove elasticsearch stack containers & volumes -docker-compose -f ./docker-compose.elasticsearch.yml down -v +docker compose -f ./docker-compose.elasticsearch.yml down -v -# Run docker-compose to start elasticsearch container -docker-compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch +# Run docker compose to start elasticsearch container +docker compose -f ./docker-compose.elasticsearch.yml up -d elasticsearch echo 'Waiting for Elasticsearch...' @@ -46,10 +46,10 @@ export ELASTICSEARCH_SERVICEACCOUNTTOKEN=$(echo $response_body | jq -r '.token.v echo 'Starting for Kibana...' -## Run docker-compose to start kibana container -docker-compose -f ./docker-compose.elasticsearch.yml up -d kibana +## Run docker compose to start kibana container +docker compose -f ./docker-compose.elasticsearch.yml up -d kibana echo 'Starting APM Server...' -## Run docker-compose to start apm-server container -docker-compose -f ./docker-compose.elasticsearch.yml up -d apm-server +## Run docker compose to start apm-server container +docker compose -f ./docker-compose.elasticsearch.yml up -d apm-server diff --git a/start-multistorage.sh b/start-multistorage.sh index 6c436de666..ad28b7b28f 100755 --- a/start-multistorage.sh +++ b/start-multistorage.sh @@ -24,8 +24,8 @@ else function down() { # Stop containers and clear volumes - docker-compose -f ./docker-compose.storage-squid.yml down -v - docker-compose down -v + docker compose -f ./docker-compose.storage-squid.yml down -v + docker compose down -v } trap down EXIT ERR SIGINT SIGTERM @@ -34,7 +34,7 @@ fi if [ "${SKIP_NODE}" != true ] then ## Run a local development chain - docker-compose up -d joystream-node + docker compose up -d joystream-node fi ## Query Node Infrastructure @@ -44,7 +44,7 @@ fi ./start-orion.sh ## Storage Squid -docker-compose -f ./docker-compose.storage-squid.yml up -d +docker compose -f ./docker-compose.storage-squid.yml up -d ## Init the chain with some state if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then @@ -58,19 +58,19 @@ if [[ $SKIP_CHAIN_SETUP != 'true' ]]; then ## Member faucet export INVITER_KEY=`cat ./tests/network-tests/output.json | jq -r .faucet.suri` - docker-compose up -d faucet + docker compose up -d faucet ## Storage Infrastructure Nodes - docker-compose up -d colossus-1 - docker-compose up -d distributor-1 - docker-compose up -d colossus-2 - docker-compose up -d distributor-2 + docker compose up -d colossus-1 + docker compose up -d distributor-1 + docker compose up -d colossus-2 + docker compose up -d distributor-2 fi if [ "${PERSIST}" == true ] then echo "All services started in the background" - echo "Remember to run 'docker-compose down -v' to kill all docker services before starting new playground." + echo "Remember to run 'docker compose down -v' to kill all docker services before starting new playground." else echo "use Ctrl+C to shutdown the development network." while true; do diff --git a/start-orion.sh b/start-orion.sh index 58edc75926..200db3b66b 100755 --- a/start-orion.sh +++ b/start-orion.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash set -e -docker-compose up -d orion-db -docker-compose up -d orion-processor -docker-compose up -d orion-graphql-api -docker-compose up -d orion-auth-api -docker-compose up -d squid-archive-ingest -docker-compose up -d squid-archive-db -docker-compose up -d squid-archive-gateway -docker-compose up -d squid-archive-explorer +docker compose up -d orion-db +docker compose up -d orion-processor +docker compose up -d orion-graphql-api +docker compose up -d orion-auth-api +docker compose up -d squid-archive-ingest +docker compose up -d squid-archive-db +docker compose up -d squid-archive-gateway +docker compose up -d squid-archive-explorer diff --git a/start.sh b/start.sh index 8fdd660394..65f0dee813 100755 --- a/start.sh +++ b/start.sh @@ -26,8 +26,8 @@ else function down() { # Stop containers and clear volumes - docker-compose -f ./docker-compose.storage-squid.yml down -v - docker-compose down -v + docker compose -f ./docker-compose.storage-squid.yml down -v + docker compose down -v } trap down EXIT ERR SIGINT SIGTERM @@ -36,7 +36,7 @@ fi if [ "${SKIP_NODE}" != true ] then ## Run a local development chain - docker-compose up -d joystream-node + docker compose up -d joystream-node fi ## Query Node Infrastructure @@ -46,7 +46,7 @@ fi ./start-orion.sh ## Storage Squid -docker-compose -f ./docker-compose.storage-squid.yml up -d +docker compose -f ./docker-compose.storage-squid.yml up -d ## Init the chain with some state if [[ $SKIP_CHAIN_SETUP != true ]]; then @@ -58,17 +58,17 @@ if [[ $SKIP_CHAIN_SETUP != true ]]; then ## Member faucet export INVITER_KEY=`cat ./tests/network-tests/output.json | jq -r .faucet.suri` - docker-compose up -d faucet + docker compose up -d faucet ## Storage Infrastructure Nodes - docker-compose up -d colossus-1 - docker-compose up -d distributor-1 + docker compose up -d colossus-1 + docker compose up -d distributor-1 fi if [ "${PERSIST}" == true ] then echo "All services started in the background" - echo "Remember to run 'docker-compose down -v' to kill all docker services before starting new playground." + echo "Remember to run 'docker compose down -v' to kill all docker services before starting new playground." else echo "use Ctrl+C to shutdown the development network." while true; do diff --git a/tests/network-tests/run-node-docker.sh b/tests/network-tests/run-node-docker.sh index 370c0df6e0..3c6914862f 100755 --- a/tests/network-tests/run-node-docker.sh +++ b/tests/network-tests/run-node-docker.sh @@ -66,7 +66,7 @@ docker run --pull never --rm -v ${DATA_PATH}:/spec joystream/node:${RUNTIME} bui # Start a chain with generated chain spec export JOYSTREAM_NODE_TAG=${RUNTIME} -docker-compose -p joystream -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec --name joystream-node \ +docker compose -p joystream -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec --name joystream-node \ --service-ports joystream-node \ --alice --validator --unsafe-ws-external --unsafe-rpc-external \ --rpc-methods Unsafe --rpc-cors=all -l runtime \ diff --git a/tests/network-tests/run-runtime-upgrade-tests.sh b/tests/network-tests/run-runtime-upgrade-tests.sh index bf3e7cac38..71af160a16 100755 --- a/tests/network-tests/run-runtime-upgrade-tests.sh +++ b/tests/network-tests/run-runtime-upgrade-tests.sh @@ -68,7 +68,7 @@ function create_raw_chain_spec() { # Start a chain with generated chain spec function start_joystream_node { - docker-compose -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec \ + docker compose -f ../../docker-compose.yml run -d -v ${DATA_PATH}:/spec \ --name joystream-node \ -p 9944:9944 -p 9933:9933 joystream-node \ --validator --unsafe-ws-external --unsafe-rpc-external \ @@ -140,7 +140,7 @@ function init_chain_db() { # if the initial state is large. # exporting should give some essential tasks errors but they are harmless https://github.com/paritytech/substrate/issues/10583 echo >&2 "exporting state" - docker-compose -f ../../docker-compose.yml run --rm \ + docker compose -f ../../docker-compose.yml run --rm \ -v ${DATA_PATH}:/spec joystream-node export-state \ --chain /spec/chain-spec-forked.json \ --base-path /data --pruning archive >${DATA_PATH}/exported-state.json diff --git a/tests/network-tests/run-tests.sh b/tests/network-tests/run-tests.sh index beae3217f0..75774fabbc 100755 --- a/tests/network-tests/run-tests.sh +++ b/tests/network-tests/run-tests.sh @@ -16,11 +16,11 @@ function cleanup() { docker logs colossus-2 --tail 100 || : if [ "${NO_STORAGE}" != true ]; then - docker-compose -f ../../docker-compose.storage-squid.yml down -v - docker-compose -f ../../docker-compose.localstack.yml down -v + docker compose -f ../../docker-compose.storage-squid.yml down -v + docker compose -f ../../docker-compose.localstack.yml down -v fi - docker-compose -f ../../docker-compose.yml down -v + docker compose -f ../../docker-compose.yml down -v } trap cleanup EXIT ERR SIGINT SIGTERM diff --git a/tests/network-tests/start-storage.sh b/tests/network-tests/start-storage.sh index 8e6eb5cffb..a76f9e5a8f 100755 --- a/tests/network-tests/start-storage.sh +++ b/tests/network-tests/start-storage.sh @@ -8,7 +8,7 @@ THIS_DIR=$(dirname $TMP) echo "Staring storage infrastructure" # Start Storage-Squid -docker-compose -f $THIS_DIR/../../docker-compose.storage-squid.yml up -d +docker compose -f $THIS_DIR/../../docker-compose.storage-squid.yml up -d HOST_IP=`$THIS_DIR/get-host-ip.sh` export COLOSSUS_1_URL="http://${HOST_IP}:3333" @@ -24,7 +24,7 @@ sleep 30 export LOCALSTACK_ENABLED=true export LOCALSTACK_HOST="${HOST_IP}:4566" export LOCALSTACK_ENDPOINT="http://${LOCALSTACK_HOST}" -docker-compose -f $THIS_DIR/../../docker-compose.localstack.yml up -d localstack && sleep 15 +docker compose -f $THIS_DIR/../../docker-compose.localstack.yml up -d localstack && sleep 15 python3 -m venv .venv pip install awscli-local source .venv/bin/activate @@ -33,10 +33,10 @@ awslocal s3api create-bucket --bucket test-bucket-2 --endpoint ${LOCALSTACK_ENDP deactivate # Start colossus & argus -docker-compose -f $THIS_DIR/../../docker-compose.yml up -d colossus-1 -docker-compose -f $THIS_DIR/../../docker-compose.yml up -d distributor-1 -docker-compose -f $THIS_DIR/../../docker-compose.yml up -d colossus-2 -docker-compose -f $THIS_DIR/../../docker-compose.yml up -d distributor-2 +docker compose -f $THIS_DIR/../../docker-compose.yml up -d colossus-1 +docker compose -f $THIS_DIR/../../docker-compose.yml up -d distributor-1 +docker compose -f $THIS_DIR/../../docker-compose.yml up -d colossus-2 +docker compose -f $THIS_DIR/../../docker-compose.yml up -d distributor-2 # allow a few seconds for nodes to startup and display first few log entries # to help debug tests diff --git a/tests/network-tests/test-setup-new-chain.sh b/tests/network-tests/test-setup-new-chain.sh index 1bc9c5710e..0ce68de5c6 100755 --- a/tests/network-tests/test-setup-new-chain.sh +++ b/tests/network-tests/test-setup-new-chain.sh @@ -18,7 +18,7 @@ function cleanup() { docker logs ${CONTAINER_ID} --tail 15 docker stop ${CONTAINER_ID} docker rm ${CONTAINER_ID} - docker-compose -f ../../docker-compose.yml down -v + docker compose -f ../../docker-compose.yml down -v } trap cleanup EXIT ERR SIGINT SIGTERM