diff --git a/docs/en_US/0.1.0/StreamisDeployment.md b/docs/en_US/0.1.0/StreamisDeployment.md new file mode 100644 index 000000000..e676ede3e --- /dev/null +++ b/docs/en_US/0.1.0/StreamisDeployment.md @@ -0,0 +1,165 @@ +# Streamis installation and deployment documentation + +## 1. Component introduction +---------- + +Streamis0.1.0 provides the Streamis-JobManager component, the role of the component is
+1. Publish streaming applications
+2. Set streaming application parameters, such as the number of Flink slots, checkpoint related parameters, etc.
+3. Manage streaming applications (e.g. start and stop)
+4. Streaming application monitoring
+ + +## 2. Code compilation +---------- +If you have already obtained the installation package, you can skip this step
+The background compilation method is as follows +``` +cd ${STREAMIS_CODE_HOME} +mvn -N install +mvn clean install +``` +The front-end compilation method is as follows +```bash +cd ${STREAMIS_CODE_HOME}/web +npm i +npm run build +``` + + +## 3. Installation preparation +### 3.1 Basic environment installation +        The following software must be installed: + +-MySQL (5.5+), [How to install MySQL](https://www.runoob.com/mysql/mysql-install.html) +-JDK (above 1.8.0_141), [How to install JDK](https://www.runoob.com/java/java-environment-setup.html) + +### 3.2Linkis environment +The execution of Streamis depends on Linkis, and it needs to be version 1.0.3 and above, so you need to install Linkis above 1.0.3 and ensure that the Flink engine can be used normally. Specifically, you can create a new and edit a flinksql script on Scriptis +And execute. If flinksql can be executed correctly, it means that the linkis1.0.3 environment is normal. + +### 3.3 Package Preparation +Obtain the software package from the third step and upload it to the installation directory of the server, such as /appcom/Install/streamis +```bash +cd /appcom/Install/streamis +tar -xvf wedatasphere-streamis-${streamis-version}-dist.tar.gz +``` + +### 3.4 Modify the database configuration +```bash +vi conf/db.sh +#Configure basic database information +``` + +### 3.5 Modify the basic configuration file + +```bash +vi conf/config.sh +#Configure service port information +#Configure Linkis service information +``` +## 4. Installation and startup +---------- + +-Background installation +```bash +sh bin/install.sh +``` + +-The install.sh script will ask you if you need to initialize the database and import metadata. + +     Because the user is worried that the user repeatedly executes the install.sh script to clear the user data in the database, when the install.sh is executed, the user will be asked if they need to initialize the database and import metadata. + +     **Yes must be selected for the first installation**. + + +- start up +```bash +sh bin/start-streamis.sh +``` + +-Start verification +Verification method, because Streamis and Linkis use a set of Eureka, you need to check whether the Eureka page of Linkis already contains Streamis services, as shown in the figure, +![components](../../images/zh_CN/eureka_streamis.png) + + + +-Front-end deployment + +1. Install nginx + +```bash +sudo yum install -y nginx +``` +2. Deploy the front-end package +``` +mkdir ${STREAMIS_FRONT_PATH} +cd ${STREAMIS_FRONT_PATH} +#Place the front-end package +unzip streamis-web.zip +``` +3. Modify the nginx configuration file
+ +```bash +cd /etc/nginx/conf.d +vi streamis.conf +# Copy the following template and modify it according to the actual situation +``` +``` +server { + listen 9088;# access port + server_name localhost; + location / { + root ${STREAMIS_FRONT_PAH}; # Please modify it to the appropriate static file directory of Streamis + index index.html index.html; + } + location /api { + proxy_pass http://${Linkis_GATEWAY_IP}:${LINKIS_GATEWY_PORT}; #Back-end Linkis address, please modify it to the ip and port of the Linkis gateway + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header x_real_ipP $remote_addr; + proxy_set_header remote_addr $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_http_version 1.1; + proxy_connect_timeout 4s; + proxy_read_timeout 600s; + proxy_send_timeout 12s; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection upgrade; + } + + #error_page 404 /404.html; + # redirect server error pages to the static page /50x.html + # + error_page 500 502 503 504 /50x.html; + location = /50x.html { + root /usr/share/nginx/html; + } +} +``` +4. Restart nginx +```bash +sudo systemctl restart nginx +``` + +## 5. Access to DSS +Streamis0.1.0 version access to DSS only implements the first-level specification, that is, you can jump to the Streamis front-end without password on the DSS page. Specifically, you need to configure the basic information of streamis into the DSS database, as follows: You need to replace the ip and port in the first line of sql. +```roomsql +SET @STREAMIS_INSTALL_IP_PORT='127.0.0.1:9088'; +SET @URL = replace('http://STREAMIS_IP_PORT','STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); +SET @HOMEPAGE_URL = replace('http://STREAMIS_IP_PORT','STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); +SET @PROJECT_URL = replace('http://STREAMIS_IP_PORT','STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); + + +delete from `dss_application` WHERE `name` ='STREAMIS'; +INSERT INTO `dss_application`(`name`,`url`,`is_user_need_init`,`level`,`user_init_url`,`exists_project_service`,`project_url`,`enhance_json`,`if_iframe`,`homepage_url`,`redirect_url`) VALUES ('STREAMIS', @URL, 0, 1, NULL, 0, @PROJECT_URL,'', 1, @HOMEPAGE_URL, @REDIRECT_URL); + +select @dss_STREAMIS_applicationId:=id from `dss_application` WHERE `name` ='STREAMIS'; + +select @dss_onestop_menu_id:=id from `dss_onestop_menu` where `name` ='Application Development'; + +delete from `dss_onestop_menu_application` WHERE title_en ='STREAMIS'; +INSERT INTO `dss_onestop_menu_application` (`application_id`, `onestop_menu_id`, `title_en`, `title_cn`, `desc_en`, `desc_cn`, `labels_en`, `labels_cn`, `is_active`, `access_button_en`, `access_button_cn`, `manual_button_en`, `manual_button_cn`, `manual_button_url`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`, `image`) +VALUES(@dss_STREAMIS_applicationId, @dss_onestop_menu_id,'STREAMIS','Real-time computing platform','Streamis is a streaming application development management system. Based on DataSphereStudio framing capabilities and the underlying docking of Linkis Flink engine, users can complete the development, debugging, release and production management of streaming applications at low cost','Streamis is a streaming application development and management system jointly built by WeBank in cooperation with Tianyi Cloud, Xian Weng Technology and Samoyed Cloud. Based on the framework capabilities of DataSphere Studio, and the underlying docking Linkis The Flink engine allows users to complete the development, debugging, release and production management of streaming applications at low cost.','Streamis','real-time computing platform','1','Enter Streamis','enter Streamis','user manual','User Manual','http://127.0.0.1:8088/wiki/scriptis/manual/workspace_cn.html','shujujiaohuan-logo',NULL,NULL,NULL,NULL,NULL,'shujujiaohuan-icon '); +``` +After executing sql, you may need to restart dss-framework-project-server to refresh the memory. \ No newline at end of file diff --git "a/docs/zh_CN/0.1.0/Streamis\345\256\211\350\243\205\346\226\207\346\241\243.md" "b/docs/zh_CN/0.1.0/Streamis\345\256\211\350\243\205\346\226\207\346\241\243.md" index 8ee85b51e..6d7409909 100644 --- "a/docs/zh_CN/0.1.0/Streamis\345\256\211\350\243\205\346\226\207\346\241\243.md" +++ "b/docs/zh_CN/0.1.0/Streamis\345\256\211\350\243\205\346\226\207\346\241\243.md" @@ -1,27 +1,16 @@ # Streamis安装部署文档 -## 1.模块介绍 +## 1.组件介绍 ---------- -Streamis有多个模块，具体如表1-1，每个模块都可以独立提供服务，也可以合并成为一个服务对外提供能力。 +Streamis0.1.0提供了Streamis-JobManager组件,组件的作用有
+1.发布流式应用
+2.设置流式应用参数,如Flink的Slot数量,checkpoint相关参数等
+3.管理流式应用(例如启停)
+4.流式应用监控
-| 模块名 | 模块作用 | 是否必须 | -|---------------------|----------------------------------------------------------------------------------------------------------------|---------------------------------------------------------| -| streamis-jobmanager | 1.发布流式应用 2.管理流式应用(例如启停) 3.设置流式应用参数 4.流式应用监控 | 1.必须，使用Streamis必须要部署jobmanager | -| streamis-datasource | 1.流式数据源管理，与Linkis数据源打通 2.流式数据源导入导出 3.流式数据源转换成Flink执行代码 4.流式数据源支持执行 | 1.如果需要使用流式工作流，则必须 2.搭建流式数仓，则必须 | -| streamis-workflow | 1.可视化编辑流式工作流 2.流式工作流执行 | 1.使用流式工作流，必须 | -| streamis-plugins | 1.与DSS集成相关组件 2.流式工作流转成流式应用 | 1.如果需要和DSS集成，则必须 | -## 2.版本选择 ----------- - -  我们将Streamis的安装版本分成两个版本,如下 - -1. 精简版，该版本只需要包含StreamisJobManager,用户可以通过上传物料包将流式应用发布到JobManager，然后对流式应用进行启停、管理和监控等，因为安装简单，可以作为一个流式应用管理平台。 - -2. 实时数仓版本。该版本会将流式工作流、jobmanager以及数据源等内容统一成一个服务，对外会提供流式工作流编辑、执行，以及一键发布到jobmanager，并且支持数据源的管理和分享，同时也支持集成到DSS。 - -## 3.代码编译 +## 2.代码编译 ---------- 如果已经获取到安装包，可以跳过此步骤
后台编译方式如下 @@ -38,39 +27,39 @@ npm run build ``` -## 4.安装准备 -### 4.1基础环境安装 +## 3.安装准备 +### 3.1基础环境安装         下面的软件必须安装： - MySQL (5.5+)，[如何安装MySQL](https://www.runoob.com/mysql/mysql-install.html) - JDK (1.8.0_141以上)，[如何安装JDK](https://www.runoob.com/java/java-environment-setup.html) -### 4.2Linkis环境 -Streamis的执行依赖于Linkis，并且需要在1.0.2以上的版本，所以您需要安装1.0.2以上的Linkis +### 3.2Linkis环境 +Streamis的执行依赖于Linkis，并且需要在1.0.3及以上的版本，所以您需要安装1.0.3以上的Linkis,并且保证Flink引擎可以正常使用,具体的,您可以在Scriptis上新建编辑一个flinksql脚本 +并执行。如果flinksql能正确执行，表示linkis1.0.3环境是正常的。 -### 4.3软件包准备 -从第三步骤中获取软件包，并上传到服务器的安装目录,如 /appcom/Install/streamis +### 3.3软件包准备 +从第二步骤中获取软件包，并上传到服务器的安装目录,如 /appcom/Install/streamis ```bash cd /appcom/Install/streamis tar -xvf wedatasphere-streamis-${streamis-version}-dist.tar.gz ``` -### 4.4修改数据库配置 +### 3.4修改数据库配置 ```bash vi conf/db.sh #配置基础的数据库信息 ``` -### 4.5修改基础配置文件 +### 3.5修改基础配置文件 ```bash vi conf/config.sh #配置服务端口信息 #配置Linkis服务信息 ``` -## 5.安装和启动 +## 4.安装和启动 ---------- -### 5.1 精简版 - 后台安装 ```bash @@ -153,7 +142,24 @@ server { sudo systemctl restart nginx ``` +## 5.接入DSS +Streamis0.1.0版本接入DSS只实现了一级规范，也就是能够在DSS的页面进行免密跳转到Streamis前端，具体的您需要将streamis的基本信息配置到DSS的数据库中，具体如下,你需要替换sql首行的ip和端口即可。 +```roomsql +SET @STREAMIS_INSTALL_IP_PORT='127.0.0.1:9088'; -- 注意此处是streamis的前端地址 +SET @URL = replace('http://STREAMIS_IP_PORT', 'STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); +SET @HOMEPAGE_URL = replace('http://STREAMIS_IP_PORT', 'STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); +SET @PROJECT_URL = replace('http://STREAMIS_IP_PORT', 'STREAMIS_IP_PORT', @STREAMIS_INSTALL_IP_PORT); + +delete from `dss_application` WHERE `name` = 'STREAMIS'; +INSERT INTO `dss_application`(`name`,`url`,`is_user_need_init`,`level`,`user_init_url`,`exists_project_service`,`project_url`,`enhance_json`,`if_iframe`,`homepage_url`,`redirect_url`) VALUES ('STREAMIS', @URL, 0, 1, NULL, 0, @PROJECT_URL, '', 1, @HOMEPAGE_URL, @REDIRECT_URL); +select @dss_STREAMIS_applicationId:=id from `dss_application` WHERE `name` = 'STREAMIS'; -### 5.2实时数仓版本 +select @dss_onestop_menu_id:=id from `dss_onestop_menu` where `name` = '应用开发'; + +delete from `dss_onestop_menu_application` WHERE title_en = 'STREAMIS'; +INSERT INTO `dss_onestop_menu_application` (`application_id`, `onestop_menu_id`, `title_en`, `title_cn`, `desc_en`, `desc_cn`, `labels_en`, `labels_cn`, `is_active`, `access_button_en`, `access_button_cn`, `manual_button_en`, `manual_button_cn`, `manual_button_url`, `icon`, `order`, `create_by`, `create_time`, `last_update_time`, `last_update_user`, `image`) +VALUES(@dss_STREAMIS_applicationId, @dss_onestop_menu_id, 'STREAMIS','实时计算平台','Streamis is a streaming application development management system. Based on DataSphereStudio framing capabilities and the underlying docking of Linkis Flink engine, users can complete the development, debugging, release and production management of streaming applications at low cost','Streamis是微众银行联合天翼云、仙翁科技和萨摩耶云共建的流式应用开发管理系统。基于DataSphereStudio的框架化能力，以及底层对接Linkis的Flink引擎，让用户低成本完成流式应用的开发、调试、发布和生产管理。','Streamis','实时计算平台','1','Enter Streamis','进入Streamis','user manual','用户手册','http://127.0.0.1:8088/wiki/scriptis/manual/workspace_cn.html','shujujiaohuan-logo',NULL,NULL,NULL,NULL,NULL,'shujujiaohuan-icon'); +``` +执行完sql之后，您可能需要重新启动一下dss-framework-project-server进行一下内存刷新。 \ No newline at end of file