diff --git a/Makefile b/Makefile index 162c2a6..c7fdcfc 100644 --- a/Makefile +++ b/Makefile @@ -26,6 +26,7 @@ dependencies: test_environment $(PYTHON_INTERPRETER) -m pip install -r requirements.minimal sudo curl -o ./assets/hadoop-aws-3.3.4.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.4/hadoop-aws-3.3.4.jar sudo curl -o ./assets/aws-java-sdk-bundle-1.12.506.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-bundle/1.12.506/aws-java-sdk-bundle-1.12.506.jar + sudo curl -o ./assets/aws-java-sdk-core-1.12.506.jar https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/1.12.506/aws-java-sdk-core-1.12.506.jar sudo curl -o ./assets/hadoop-common-3.3.4.jar https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-common/3.3.4/hadoop-common-3.3.4.jar sudo curl -o ./assets/postgresql-42.6.0.jar https://jdbc.postgresql.org/download/postgresql-42.6.0.jar #cp ./assets/hadoop-aws-3.3.4.jar ~/$(CONDA_FOLDER_NAME)/envs/next-watch/lib/python3.10/site-packages/pyspark/jars/ @@ -56,6 +57,7 @@ run: ## Populate Databse with Users from production datasets users: + docker compose exec dev-spark bash -c "python3.9 src/main.py -p 'de'" docker compose exec dev-spark bash -c "python3.9 src/scripts/populate_db_with_users.py" ## Run DE pipelines diff --git a/README.md b/README.md index 1a77ad1..98106a5 100644 --- a/README.md +++ b/README.md @@ -28,31 +28,45 @@ git clone https://github.com/brnaguiar/mlops-next-watch.git make env ``` -3. Install requirements / dependencies and assets +3. Activate conda env +```sh +source activate nwenv +``` + +4. Install requirements / dependencies and assets ```sh make dependencies ``` -4. Pull the datasets +5. Pull the datasets ```sh make datasets ``` -5. Configure containers and secrets +6. Configure containers and secrets ```sh make init ``` -6. Run Docker Compose +7. Run Docker Compose ```sh make run ``` -7. Populate production Database with users +8. Populate production Database with users ```sh make users ``` +## Useful Service Endpoints +``` +- Jupyter `http://localhost:8888` +- Minio `http://localhost:9001` +- MLFlow `http://localhost:5000` +- FastAPI `http://localhost:8086/` +- Streamlit UI `http://localhost:8501` +- Grafana Dashboard `http://localhost:3000` +``` ## Architecture