Merge pull request #410 from WeDataSphere/dev-1.1.2

Merge the latest code into master
WeBankFinTech · Aug 28, 2023 · 52caf3f · 52caf3f
2 parents 757cdc6 + 3e73acb
commit 52caf3f
Show file tree

Hide file tree

Showing 725 changed files with 25,297 additions and 12,020 deletions.
diff --git a/.github/workflows/dead-link-checker.yml b/.github/workflows/dead-link-checker.yml
@@ -1,6 +1,6 @@
 name: Dead Link Check
 
-on: [push, pull_request]
+on: [push]
 
 jobs:
   dead-links-check:

diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
 FROM harbor.local.hching.com/library/jdk:8u301
 
-ADD assembly-package/target/wedatasphere-exchangis-1.0.0-RC1.tar.gz /opt/wedatasphere-exchangis.tar.gz
+ADD assembly-package/target/wedatasphere-exchangis-1.1.2.tar.gz /opt/wedatasphere-exchangis.tar.gz
 
-RUN cd /opt/wedatasphere-exchangis.tar.gz/packages/ && tar -zxf exchangis-server_1.0.0-RC1.tar.gz && cd /opt/wedatasphere-exchangis.tar.gz/sbin
+RUN cd /opt/wedatasphere-exchangis.tar.gz/packages/ && tar -zxf exchangis-server_1.1.2.tar.gz && cd /opt/wedatasphere-exchangis.tar.gz/sbin
 
 WORKDIR /opt/wedatasphere-exchangis.tar.gz/sbin
 

diff --git a/README-ZH.md b/README-ZH.md
@@ -6,7 +6,7 @@
 
 ## 介绍
 
-Exchangis 1.0.0 是微众银行大数据平台 WeDataSphere 与社区用户共同研发的的新版数据交换工具，支持异构数据源之间的结构化和非结构化数据传输同步。
+Exchangis是微众银行大数据平台 WeDataSphere 与社区用户共同研发的的新版数据交换工具，支持异构数据源之间的结构化和非结构化数据传输同步。
 
 Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许用户快速接入新的数据源，并只需在数据库中简单配置即可在页面中使用。
 
@@ -16,7 +16,7 @@ Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许
 
 ### 界面预览
 
-![image](https://user-images.githubusercontent.com/27387830/171488936-2cea3ee9-4ef7-4309-93e1-e3b697bd3be1.png)
+![image](images/zh_CN/ch1/frontend_view.png)
 
 ## 核心特点
 
@@ -42,19 +42,22 @@ Exchangis 抽象了一套统一的数据源和同步作业定义插件，允许
 ### 3.  与DSS工作流打通，一站式大数据开发的门户
 
 - 实现DSS AppConn包括一级 SSO 规范，二级组织结构规范，三级开发流程规范在内的三级规范；
-
 - 作为DSS工作流的数据交换节点，是整个工作流链路中的门户流程，为后续的工作流节点运行提供稳固的数据基础；
 
+### 4.  支持多种导数引擎
+
+- 支持Sqoop和DataX引擎进行多种异构数据源之间的导数
+
 ## 整体设计
 
 ### 架构设计
 
-![架构设计](https://user-images.githubusercontent.com/27387830/173026793-f1475803-9f85-4478-b566-1ad1d002cd8a.png)
+![架构设计](images/zh_CN/ch1/home_page_zh.png)
 
 
 ## 相关文档
-[安装部署文档](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_deploy_cn.md)  
-[用户手册](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/zh_CN/ch1/exchangis_user_manual_cn.md)
+[安装部署文档](docs/zh_CN/ch1/exchangis_deploy_cn.md)  
+[用户手册](docs/zh_CN/ch1/exchangis_user_manual_cn.md)
 
 ## 交流贡献
 

diff --git a/README.md b/README.md
@@ -4,7 +4,7 @@ English | [中文](README-ZH.md)
 
 ## Introduction
 
-Exchangis 1.0.0 is a new version of data exchange tool jointly developed by WeDataSphere, a big data platform of WeBank, and community users, which supports the synchronization of structured and unstructured data transmission between heterogeneous data sources.
+Exchangis is a new version of data exchange tool jointly developed by WeDataSphere, a big data platform of WeBank, and community users, which supports the synchronization of structured and unstructured data transmission between heterogeneous data sources.
 
 Exchangis abstracts a unified set of data source and synchronization job definition plugins, allowing users to quickly access new data sources and use them on pages with simple configuration in the database.
 
@@ -14,7 +14,7 @@ With the help of [Linkis](https://github.com/apache/incubator-linkis) computing
 
 ###  Interface preview 
 
-![image](https://user-images.githubusercontent.com/27387830/171488936-2cea3ee9-4ef7-4309-93e1-e3b697bd3be1.png)
+![image](images/zh_CN/ch1/frontend_view.png)
 
 ## Core characteristics
 
@@ -47,13 +47,13 @@ With the help of [Linkis](https://github.com/apache/incubator-linkis) computing
 
 ### Architecture Design
 
-![架构设计](images/en_US/ch1/architecture.png)
+![架构设计](images/zh_CN/ch1/home_page_en.png)
 
 
 ## Documents
 
-[Quick Deploy](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_deploy_en.md)  
-[User Manual](https://github.com/WeBankFinTech/Exchangis/blob/dev-1.0.0/docs/en_US/ch1/exchangis_user_manual_en.md)
+[Quick Deploy](docs/en_US/ch1/exchangis_deploy_en.md)  
+[User Manual](docs/en_US/ch1/exchangis_user_manual_en.md)
 
 ## Communication and contribution 
 

diff --git a/...ackage/config/exchangis-server.properties → ...ge/config/dss-exchangis-server.properties b/...ackage/config/exchangis-server.properties → ...ge/config/dss-exchangis-server.properties
@@ -15,14 +15,15 @@
 #
 #
 
-wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true
-
+wds.linkis.test.mode=false
+wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true&useAffectedRows=true
 wds.linkis.server.mybatis.datasource.username={username}
-
 wds.linkis.server.mybatis.datasource.password={password}
 
+wds.linkis.gateway.ip={LINKIS_IP}
+wds.linkis.gateway.port={LINKIS_PORT}
+wds.linkis.gateway.url=http://{LINKIS_IP}:{LINKIS_PORT}/
 wds.linkis.log.clear=true
-
 wds.linkis.server.version=v1
 
 # datasource client
@@ -34,28 +35,33 @@ wds.exchangis.datasource.client.dws.version=v1
 # launcher client
 wds.exchangis.client.linkis.server-url=http://{LINKIS_IP}:{LINKIS_PORT}/
 wds.exchangis.client.linkis.token.value=EXCHANGIS-AUTH
-wds.exchangis.datasource.extension.dir=exchangis-extds
+wds.exchangis.datasource.extension.dir=exchangis-extds/
 
 ##restful
 wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.datasource.server.restful.api,\
   com.webank.wedatasphere.exchangis.project.server.restful,\
   com.webank.wedatasphere.exchangis.job.server.restful
-wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\
-classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\
-classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml
+
+wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\
+classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml,\
+classpath*:com/webank/wedatasphere/exchangis/project/provider/mapper/impl/*.xml,\
+classpath*:com/webank/wedatasphere/exchangis/engine/server/mapper/*.xml
 
 wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\
   com.webank.wedatasphere.exchangis.project.server.mapper,\
+  com.webank.wedatasphere.exchangis.project.provider.mapper,\
   com.webank.wedatasphere.linkis.configuration.dao,\
-  com.webank.wedatasphere.dss.framework.appconn.dao,\
-  com.webank.wedatasphere.dss.workflow.dao,\
   com.webank.wedatasphere.linkis.metadata.dao,\
   com.webank.wedatasphere.exchangis.job.server.mapper,\
-  com.webank.wedatasphere.exchangis.job.server.dao
+  com.webank.wedatasphere.exchangis.job.server.dao,\
+  com.webank.wedatasphere.exchangis.engine.dao
 
 wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.min-occupy=0.25
 wds.exchangis.job.task.scheduler.load-balancer.flexible.segments.max-occupy=0.5
 #wds.exchangis.job.scheduler.group.max.running-jobs=4
 
 wds.linkis-session.ticket.key=bdp-user-ticket-id
+wds.exchangis.limit.interface.value=false
 
+wds.exchangis.publicKeyStr=
+wds.exchangis.privateKeyStr=
diff --git a/assembly-package/config/exchangis.properties b/assembly-package/config/exchangis.properties
diff --git a/assembly-package/config/log4j2.xml b/assembly-package/config/log4j2.xml
@@ -26,13 +26,22 @@
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
         </Console>
         <RollingFile name="RollingFile" fileName="${sys:log.path}/${sys:serviceName}.log"
-                     filePattern="${sys:log.path}/$${date:yyyy-MM}/${sys:serviceName}/exchangis-log-%d{yyyy-MM-dd}-%i.log">
+                     filePattern="${sys:log.path}/$${date:yyyy-MM-dd}/${sys:serviceName}/exchangis-log-%d{yyyy-MM-dd}-%i.log">
             <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
             <SizeBasedTriggeringPolicy size="100MB"/>
             <DefaultRolloverStrategy max="20"/>
         </RollingFile>
+        <RollingFile name="AuditLog" fileName="${sys:log.path}/${sys:serviceName}-audit.log"
+                     filePattern="${sys:log.path}/$${date:yyyy-MM}/${sys:serviceName}/exchangis-audit-log-%d{yyyy-MM-dd}-%i.log">
+            <PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} [%-5level] [%-40t] %c{1.} (%L) [%M] - %msg%xEx%n"/>
+            <SizeBasedTriggeringPolicy size="1G"/>
+            <DefaultRolloverStrategy max="200"/>
+        </RollingFile>
     </appenders>
     <loggers>
+        <logger name="com.webank.wedatasphere.exchangis.job.utils.AuditLogUtils" level="INFO" additivity="false">
+            <appender-ref ref="AuditLog"/>
+        </logger>
         <root level="INFO" additivity="false">
             <appender-ref ref="RollingFile"/>
             <appender-ref ref="Console"/>

diff --git a/...bly-package/config/application-eureka.yml → ...-processor-templates/datax-processor.java b/...bly-package/config/application-eureka.yml → ...-processor-templates/datax-processor.java
diff --git a/assembly-package/pom.xml b/assembly-package/pom.xml
@@ -21,7 +21,7 @@
 	<parent>
 		<artifactId>exchangis</artifactId>
 		<groupId>com.webank.wedatasphere.exchangis</groupId>
-		<version>1.0.0</version>
+		<version>1.1.2</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>assembly-package</artifactId>
@@ -31,13 +31,15 @@
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-install-plugin</artifactId>
+				<version>2.4</version>
 				<configuration>
 					<skip>true</skip>
 				</configuration>
 			</plugin>
 			<plugin>
 				<groupId>org.apache.maven.plugins</groupId>
 				<artifactId>maven-antrun-plugin</artifactId>
+				<version>1.3</version>
 				<executions>
 					<execution>
 						<phase>package</phase>

diff --git a/assembly-package/sbin/common.sh b/assembly-package/sbin/common.sh
@@ -16,4 +16,4 @@
 #
 
 declare -A MODULE_MAIN_CLASS
-MODULE_MAIN_CLASS["exchangis-server"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication"
+MODULE_MAIN_CLASS["dss-exchangis-main-server-dev"]="com.webank.wedatasphere.exchangis.server.boot.ExchangisServerApplication"
diff --git a/assembly-package/sbin/daemon.sh b/assembly-package/sbin/daemon.sh
@@ -23,9 +23,8 @@ else
   source ./common.sh
 fi
 
-MODULE_NAME=""
 usage(){
-  echo "Usage is [start|stop|restart {service}]"
+  echo "Usage is [start|stop|restart {server}]"
 }
 
 start(){
@@ -41,20 +40,22 @@ stop(){
 restart(){
   launcher_stop $1 $2
   if [[ $? -eq 0 ]]; then
-    sleep 2
+    sleep 3
     launcher_start $1 $2
   fi
 }
 
 COMMAND=$1
 case $COMMAND in
   start|stop|restart)
+    load_env_definitions ${ENV_FILE}
     if [[ ! -z $2 ]]; then
-      MAIN_CLASS=${MODULE_MAIN_CLASS[${MODULE_DEFAULT_PREFIX}$2]}
+      SERVICE_NAME=${MODULE_DEFAULT_PREFIX}$2${MODULE_DEFAULT_SUFFIX}
+      MAIN_CLASS=${MODULE_MAIN_CLASS[${SERVICE_NAME}]}
       if [[ "x"${MAIN_CLASS} != "x" ]]; then
-        $COMMAND ${MODULE_DEFAULT_PREFIX}$2 ${MAIN_CLASS}
+        $COMMAND ${SERVICE_NAME} ${MAIN_CLASS}
       else
-        LOG ERROR "Cannot find the main class for [ ${MODULE_DEFAULT_PREFIX}$2 ]"
+        LOG ERROR "Cannot find the main class for [ ${SERVICE_NAME} ]"
       fi
     else
       usage

diff --git a/assembly-package/sbin/env.properties b/assembly-package/sbin/env.properties
@@ -0,0 +1,4 @@
+EXCHANGIS_CONF_PATH=/appcom/config/exchangis-config/background
+EXCHANGIS_LOG_PATH=/appcom/logs/exchangis/background
+MODULE_DEFAULT_PREFIX="dss-exchangis-main-"
+MODULE_DEFAULT_SUFFIX="-dev"
diff --git a/assembly-package/sbin/install.sh b/assembly-package/sbin/install.sh
@@ -123,57 +123,36 @@ interact_echo(){
   done
 }
 
+# Initalize database
 init_database(){
-BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties"
-# Start to initalize database
-if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then
-   `mysql --version >/dev/null 2>&1`
-   interact_echo "Do you want to initalize database with sql?"
-   if [ $? == 0 ]; then
-      LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m"
-      #interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?"
-      #if [ $? == 0 ]; then
+    BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-server.properties"
+    if [ "x${SQL_SOURCE_PATH}" != "x" ] && [ -f "${SQL_SOURCE_PATH}" ]; then
+        `mysql --version >/dev/null 2>&1`
         DATASOURCE_URL="jdbc:mysql:\/\/${MYSQL_HOST}:${MYSQL_PORT}\/${DATABASE}\?useSSL=false\&characterEncoding=UTF-8\&allowMultiQueries=true"
-        mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD}  --default-character-set=utf8 -e \
-        "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};"
-        #sed -ri "s![#]?(DB_HOST=)\S*!\1${HOST}!g" ${BOOTSTRAP_PROP_FILE}
-        #sed -ri "s![#]?(DB_PORT=)\S*!\1${PORT}!g" ${BOOTSTRAP_PROP_FILE}
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.username=)\S*!\1${MYSQL_USERNAME}!g" ${BOOTSTRAP_PROP_FILE}
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.password=)\S*!\1${MYSQL_PASSWORD}!g" ${BOOTSTRAP_PROP_FILE}
         sed -ri "s![#]?(wds.linkis.server.mybatis.datasource.url=)\S*!\1${DATASOURCE_URL}!g" ${BOOTSTRAP_PROP_FILE}
-      #fi
-   fi
-fi
+        interact_echo "Do you want to initalize database with sql: [${SQL_SOURCE_PATH}]?"
+        if [ $? == 0 ]; then
+          LOG INFO "\033[1m Scan out mysql command, so begin to initalize the database\033[0m"
+          mysql -h ${MYSQL_HOST} -P ${MYSQL_PORT} -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD}  --default-character-set=utf8 -e \
+          "CREATE DATABASE IF NOT EXISTS ${DATABASE}; USE ${DATABASE}; source ${SQL_SOURCE_PATH};"
+        fi
+    fi
 }
 
 init_properties(){
-BOOTSTRAP_PROP_FILE="${CONF_PATH}/exchangis-server.properties"
-APPLICATION_YML="${CONF_PATH}/application-exchangis.yml"
-# Start to initalize propertis
-      #interact_echo "Do you want to initalize exchangis-server.properties?"
-      #if [ $? == 0 ]; then
-
-        LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/"
-
-        if [ "x${LINKIS_SERVER_URL}" == "x" ]; then
-          LINKIS_SERVER_URL="http://127.0.0.1:3306"
-        fi
-        if [ "x${LINKIS_SERVER_URL}" == "x" ]; then
-          LINKIS_SERVER_URL="http://127.0.0.1:3306"
-        fi
+    BOOTSTRAP_PROP_FILE="${CONF_PATH}/dss-exchangis-server.properties"
+    APPLICATION_YML="${CONF_PATH}/application-exchangis.yml"
+    LINKIS_GATEWAY_URL="http:\/\/${LINKIS_GATEWAY_HOST}:${LINKIS_GATEWAY_PORT}\/"
+    if [ "x${LINKIS_SERVER_URL}" == "x" ]; then
+      LINKIS_SERVER_URL="http://127.0.0.1:9001"
+    fi
 
-        sed -ri "s![#]?(wds.linkis.gateway.ip=)\S*!\1${LINKIS_GATEWAY_HOST}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(wds.linkis.gateway.url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE}
-        #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.key=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE}
-        #sed -ri "s![#]?(wds.exchangis.datasource.client.authtoken.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE}
-        #sed -ri "s![#]?(wds.exchangis.client.linkis.token.value=)\S*!\1${LINKIS_TOKEN}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(wds.linkis.gateway.port=)\S*!\1${LINKIS_GATEWAY_PORT}!g" ${BOOTSTRAP_PROP_FILE}
-        sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML}
-        sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML}
-      #fi
+    sed -ri "s![#]?(wds.exchangis.datasource.client.serverurl=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE}
+    sed -ri "s![#]?(wds.exchangis.client.linkis.server-url=)\S*!\1${LINKIS_GATEWAY_URL}!g" ${BOOTSTRAP_PROP_FILE}
+    sed -ri "s![#]?(port: )\S*!\1${EXCHANGIS_PORT}!g" ${APPLICATION_YML}
+    sed -ri "s![#]?(defaultZone: )\S*!\1${EUREKA_URL}!g" ${APPLICATION_YML}
 }
 
 install_modules(){