Merge pull request #125 from MichaelPesce/update-tests

Expand E2E testing, fix logging panel bug

.github/workflows/electron-build.yml

@@ -19,15 +19,13 @@ jobs:
 
     runs-on: windows-latest
 
-    # strategy:
-    #   fail-fast: false
-
     steps:
       - uses: actions/checkout@v4
       - uses: conda-incubator/setup-miniconda@v3
         with:
           environment-file: environment-build.yml
           activate-environment: watertap-ui-env
+          miniforge-version: latest
 
       - name: Install Azure Signtool
         run: |
@@ -44,18 +42,6 @@ jobs:
       - name: Get idaes extensions
         run: idaes get-extensions
 
-      # - name: Install Watertap locally
-      #   working-directory: ../
-      #   run: git clone https://github.com/MichaelPesce/watertap.git && cd watertap && git fetch --all && git checkout update-pydantic && pip install --progress-bar off .
-
-      # - name: Install Watertap locally
-      #   working-directory: ../
-      #   run: git clone https://github.com/watertap-org/watertap.git && cd watertap && pip install --progress-bar off .
-
-      # - name: Install Watertap locally
-      #   working-directory: ../
-      #   run: git clone https://github.com/watertap-org/watertap.git && cd watertap && git fetch --all --tags && git checkout 0.12.0 && pip install --progress-bar off .
-
       - name: Install adudchenko/generic treatment train Watertap
         working-directory: ../
         run: git clone https://github.com/avdudchenko/watertap.git && cd watertap && git fetch --all --tags && git checkout generic_treatment_train && pip install --progress-bar off .
@@ -68,20 +54,6 @@ jobs:
         run: |
           pip install --progress-bar off .
 
-      # install slightly older versions of numpy, pandas, and scipy. 
-      # having issues when freezing the latest packages of each (numpy 1.26, pandas 2.1, scipy 1.11)
-      # - name: Install numpy 1.24.3
-      #   run: |
-      #     pip uninstall -y numpy && pip install numpy==1.24.3
-
-      # - name: Install pandas 2.0.3
-      #   run: |
-      #     pip uninstall -y pandas && pip install pandas==2.0.3
-
-      # - name: Install scipy 1.9.1
-      #   run: |
-      #     pip uninstall -y scipy && pip install scipy==1.9.1
-
       - name: Build Backend
         run: npm --prefix electron run build-backend
 
@@ -103,76 +75,3 @@ jobs:
             name: windows-dist
             path: |
               electron/dist/WaterTAP-UI_24.05.17_win64.exe
-
-
-  # linux-build:
-  #   name: Linux Build
-
-  #   runs-on: ubuntu-latest
-
-  #   env: 
-  #     CI: false
-
-  #   steps:
-  #     - uses: actions/checkout@v3
-  #     - uses: conda-incubator/setup-miniconda@v2
-  #       with:
-  #         environment-file: environment.yml
-  #         activate-environment: watertap-ui-env
-
-  #     - name: Install Electron JS dependencies
-  #       run: |
-  #         npm --prefix electron clean-install
-
-  #     - name: Install frontend JS dependencies
-  #       run: |
-  #         npm --prefix electron/ui clean-install
-
-  #     - name: Install GNU FORTRAN, GOMP, Blas, and Lapack libraries
-  #       run: |
-  #         sudo apt-get install -y libgfortran5 libgomp1 liblapack3 libblas3
-
-  #     - name: Get idaes extensions
-  #       run: idaes get-extensions
-
-  #     - name: Install Watertap locally
-  #       working-directory: ../
-  #       run: git clone https://github.com/watertap-org/watertap.git && cd watertap && pip install --progress-bar off .
-
-  #     - name: Transfer Entry points
-  #       run: |
-  #         npm --prefix electron run move-entrypoints
-
-  #     - name: Install watertap-ui Python package
-  #       run: |
-  #         pip install --progress-bar off .
-
-  #     # - name: Create idaes extensions installer
-  #     #   run: |
-  #     #     npm --prefix electron run get-extensions-installer
-
-  #     # - name: Revert to correct idaes version
-  #     #   run: |
-  #     #     pip uninstall -y idaes-pse && pip install 'idaes-pse @ https://github.com/IDAES/idaes-pse/archive/2.0.0a3.zip'
-
-  #     - name: Build Backend
-  #       run: npm --prefix electron run build-backend
-
-  #     - name: Build Frontend
-  #       run: npm --prefix electron run build-frontend
-
-  #     # for some reason, ico file breaks linux build. going without icon for now
-  #     - name: Remove ico file
-  #       run: rm electron/build/nawi-logo.ico
-
-  #     - name: Build Linux Distribution
-  #       run: npm --prefix electron run electron-build-lin
-
-  #     - name: Upload artifact for linux build
-  #       uses: actions/upload-artifact@v3
-  #       if: always()
-  #       with:
-  #           name: linux-dist
-  #           path: |
-  #             electron/dist/watertap-ui_0.1.0_amd64.deb
-

.github/workflows/main.yml

@@ -40,6 +40,7 @@ jobs:
         with:
           environment-file: environment.yml
           activate-environment: watertap-ui-env
+          miniforge-version: latest
 
       - name: Install Electron JS dependencies
         run: |
@@ -56,28 +57,10 @@ jobs:
         run: |
           pip install --progress-bar off .
 
-      # - name: Install watertap from yet unmerged PR
-      #   run: |
-      #     pip install git+https://github.com/watertap-org/watertap@refs/pull/1270/merge
-
-      # - name: Install my fork of watertap
-      #   working-directory: ../
-      #   run: git clone https://github.com/MichaelPesce/watertap.git && cd watertap && git fetch --all && git checkout build-options && pip install --progress-bar off .
-
-      # - name: Install correct branch of watertap
-      #   working-directory: ../
-      #   run: git clone https://github.com/MichaelPesce/watertap.git && cd watertap && pip install --progress-bar off .
-
       - name: Display installed environment
         run: |
           conda env export --no-builds
 
-      # - name: run pytest
-      #   working-directory: backend
-      #   env:
-      #     PYTHONPATH: ${{ env.GITHUB_WORKSPACE }}/backend
-      #   run: pytest tests -v
-
       - name: run backend server
         run: uvicorn --app-dir backend/app main:app --reload --host 127.0.0.1 --port 8001 & sleep 30
 
@@ -124,6 +107,7 @@ jobs:
         with:
           environment-file: environment.yml
           activate-environment: watertap-ui-env
+          miniforge-version: latest
 
       - name: Install watertap-ui Python package
         run: |
@@ -133,10 +117,6 @@ jobs:
         run: |
           pip install httpx
 
-      # - name: Install my fork of watertap
-      #   working-directory: ../
-      #   run: git clone https://github.com/MichaelPesce/watertap.git && cd watertap && git fetch --all && git checkout update-pydantic && pip install --progress-bar off .
-
       - name: get idaes extensions
         run: idaes get-extensions
 

backend/app/internal/flowsheet_manager.py

@@ -206,7 +206,7 @@ def get_diagram(self, id_: str) -> bytes:
         if info.custom:
             # do this
             data_path = (
-                Path.home() / ".watertap" / "custom_flowsheets" / f"{info.id_}.png"
+                self.app_settings.custom_flowsheets_dir / f"{info.id_}.png"
             )
             data = data_path.read_bytes()
 
@@ -458,9 +458,19 @@ def _get_flowsheet_interfaces(
 
     def add_custom_flowsheet(self, new_files, new_id):
         """Add new custom flowsheet to the mini db."""
+        for f in new_files:
+            if "_ui.py" in f:
+                module_name = f.replace(".py", "")
+                try:
+                    importlib.import_module(module_name)
+                except Exception as e:
+                    _log.info(f"unable to import module: {e}")
+                    self.remove_custom_flowsheet_files(new_files)
+                    return e
 
         query = tinydb.Query()
         try:
+
             custom_flowsheets_dict = self._histdb.search(
                 query.fragment({"custom_flowsheets_version": VERSION})
             )
@@ -486,6 +496,7 @@ def add_custom_flowsheet(self, new_files, new_id):
         )
 
         self.add_custom_flowsheets()
+        return "success"
 
     def remove_custom_flowsheet(self, id_):
         """Remove a custom flowsheet from the mini db."""
@@ -508,15 +519,19 @@ def remove_custom_flowsheet(self, id_):
 
         # remove each file
         flowsheet_files = custom_flowsheets_dict[id_]
-        for flowsheet_file in flowsheet_files:
-            flowsheet_file_path = self.custom_flowsheets_path / flowsheet_file
-            _log.info(f"flowsheet file path: {flowsheet_file_path}")
-            if os.path.isfile(flowsheet_file_path):
-                _log.info(f"removing file: {flowsheet_file_path}")
-                os.remove(flowsheet_file_path)
+        self.remove_custom_flowsheet_files(flowsheet_files)
+        # for flowsheet_file in flowsheet_files:
+        #     flowsheet_file_path = self.custom_flowsheets_path / flowsheet_file
+        #     _log.info(f"flowsheet file path: {flowsheet_file_path}")
+        #     if os.path.isfile(flowsheet_file_path):
+        #         _log.info(f"removing file: {flowsheet_file_path}")
+        #         os.remove(flowsheet_file_path)
 
         # delete from DB
-        del custom_flowsheets_dict[id_]
+        try:
+            del custom_flowsheets_dict[id_]
+        except Exception as e:
+            _log.info(f"unable to delete {id_} from custom flowsheets dictionary")
         self._histdb.upsert(
             {
                 "custom_flowsheets_version": VERSION,
@@ -526,9 +541,22 @@ def remove_custom_flowsheet(self, id_):
         )
 
         # remove from flowsheets list
-        del self._flowsheets[id_]
+        try:
+            del self._flowsheets[id_]
+        except Exception as e:
+            _log.info(f"unable to delete {id_} from flowsheets list")
 
         self.add_custom_flowsheets()
+
+    def remove_custom_flowsheet_files(self, flowsheet_files):
+        # remove each file
+        for flowsheet_file in flowsheet_files:
+            flowsheet_file_path = self.custom_flowsheets_path / flowsheet_file
+            _log.info(f"flowsheet file path: {flowsheet_file_path}")
+            if os.path.isfile(flowsheet_file_path):
+                _log.info(f"removing file: {flowsheet_file_path}")
+                os.remove(flowsheet_file_path)
+        return
 
     def add_custom_flowsheets(self):
         """Search for user uploaded flowsheets. If found, add them as flowsheet interfaces."""
@@ -585,7 +613,7 @@ def set_number_of_subprocesses(self, value):
 
     def get_logs_path(self):
         """Return logs path."""
-        return self.app_settings.log_dir
+        return self.app_settings.log_dir / "nawi-ui_backend_logs.log"
 
     @staticmethod
     def _get_flowsheet_interface(module: ModuleType) -> Optional[FlowsheetInterface]:

backend/app/internal/get_extensions.py

@@ -4,7 +4,7 @@
 from shutil import copytree
 import certifi
 
-idaes_extensions_dir = Path.home() / ".watertap" / ".idaes"
+idaes_extensions_dir = Path.home() / ".nawi" / ".idaes"
 
 
 def check_for_idaes_extensions():

backend/app/internal/parameter_sweep.py

@@ -165,7 +165,7 @@ def run_parameter_sweep(flowsheet, info):
                 )
                 keys.append(key)
         output_path = (
-            Path.home() / ".watertap" / "sweep_outputs" / f"{info.name}_sweep.csv"
+            Path.home() / ".nawi" / "sweep_outputs" / f"{info.name}_sweep.csv"
         )
         results = run_analysis(
             m=flowsheet.fs_exp.m,

backend/app/routers/flowsheets.py

@@ -290,7 +290,6 @@ async def upload_flowsheet(files: List[UploadFile]) -> str:
     Returns:
         Updated flowsheet list
     """
-    custom_flowsheets_path = Path.home() / ".watertap" / "custom_flowsheets"
     try:
         # get file contents
         new_files = []
@@ -301,12 +300,15 @@ async def upload_flowsheet(files: List[UploadFile]) -> str:
             if "_ui.py" in file.filename:
                 new_id = file.filename.replace(".py", "")
             async with aiofiles.open(
-                f"{str(custom_flowsheets_path)}/{file.filename}", "wb"
+                f"{str(flowsheet_manager.app_settings.custom_flowsheets_dir)}/{file.filename}", "wb"
             ) as out_file:
                 content = await file.read()  # async read
                 await out_file.write(content)
-        flowsheet_manager.add_custom_flowsheet(new_files, new_id)
-        return {"return": "success boy"}
+        resp = flowsheet_manager.add_custom_flowsheet(new_files, new_id)
+        if resp == "success":
+            return new_id
+        else:
+            raise HTTPException(400, detail=f"Flowsheet Module not valid: {resp}")
 
     except Exception as e:
         _log.error(f"error on file upload: {str(e)}")
@@ -518,7 +520,7 @@ async def get_logs() -> List:
     Returns:
         Logs formatted as a list
     """
-    logs_path = flowsheet_manager.get_logs_path() / "watertap-ui_backend_logs.log"
+    logs_path = flowsheet_manager.get_logs_path()
     return parse_logs(logs_path, flowsheet_manager.startup_time)
 
 
@@ -540,5 +542,5 @@ async def download_logs() -> Path:
     Returns:
         Log file
     """
-    logs_path = flowsheet_manager.get_logs_path() / "watertap-ui_backend_logs.log"
+    logs_path = flowsheet_manager.get_logs_path()
     return logs_path

backend/app/setup-extensions.py

@@ -5,8 +5,8 @@
 from shutil import copytree
 import certifi
 
-idaes_extensions_dir = Path.home() / ".watertap" / ".idaes"
-pyomo_extensions_dir = Path.home() / ".watertap" / ".pyomo_"
+idaes_extensions_dir = Path.home() / ".nawi" / ".idaes"
+pyomo_extensions_dir = Path.home() / ".nawi" / ".pyomo_"
 def check_for_idaes_extensions():
     print('checking for idaes extensions')
     found_extensions = os.path.exists(idaes_extensions_dir)

backend/tests/app/internal/test_flowsheet_manager.py

@@ -9,10 +9,6 @@
 import time
 
 
-# logging.getLogger("idaes.app.internal.flowsheet_manager").setLevel(logging.DEBUG)
-# logging.getLogger("idaes.watertap.ui.fsapi").setLevel(logging.DEBUG)
-
-
 @pytest.fixture(scope="module")
 def mgr():
     return fm.FlowsheetManager(packages=["watertap", "not.here"])

electron/ui/cypress.config.js

@@ -1,12 +1,13 @@
 const { defineConfig } = require("cypress");
+const {downloadFile} = require('cypress-downloadfile/lib/addPlugin');
 
 module.exports = defineConfig({
   e2e: {
     setupNodeEvents(on, config) {
       // implement node event listeners here
-
+      on('task', {downloadFile});
     },
     baseUrl: 'http://localhost:3000',
     video: true
   },
-});
+});