Skip to content

Commit

Permalink
Enhance the ddl and dml sql;Fix config file
Browse files Browse the repository at this point in the history
  • Loading branch information
jefftlin committed Aug 14, 2023
1 parent 9441467 commit afdb9de
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 238 deletions.
11 changes: 5 additions & 6 deletions ...ackage/config/exchangis-server.properties → ...ge/config/dss-exchangis-server.properties
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,15 @@
#
#

wds.linkis.test.mode=false
wds.linkis.server.mybatis.datasource.url=jdbc:mysql://{IP}:{PORT}/{database}?useSSL=false&characterEncoding=UTF-8&allowMultiQueries=true&useAffectedRows=true
wds.linkis.server.mybatis.datasource.username={username}
wds.linkis.server.mybatis.datasource.password={password}

wds.linkis.gateway.ip={LINKIS_IP}
wds.linkis.gateway.port={LINKIS_PORT}
wds.linkis.gateway.url=http://{LINKIS_IP}:{LINKIS_PORT}/
wds.linkis.log.clear=true

wds.linkis.server.version=v1

# datasource client
Expand All @@ -39,9 +42,7 @@ wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.exchangis.dataso
com.webank.wedatasphere.exchangis.project.server.restful,\
com.webank.wedatasphere.exchangis.job.server.restful

wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/framework/appconn/dao/impl/*.xml,\
classpath*:com/webank/wedatasphere/dss/workflow/dao/impl/*.xml,\
classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\
wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/exchangis/job/server/mapper/impl/*.xml,\
classpath*:com/webank/wedatasphere/exchangis/project/server/mapper/impl/*.xml,\
classpath*:com/webank/wedatasphere/exchangis/project/provider/mapper/impl/*.xml,\
classpath*:com/webank/wedatasphere/exchangis/engine/server/mapper/*.xml
Expand All @@ -50,8 +51,6 @@ wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.exchangis.dao,\
com.webank.wedatasphere.exchangis.project.server.mapper,\
com.webank.wedatasphere.exchangis.project.provider.mapper,\
com.webank.wedatasphere.linkis.configuration.dao,\
com.webank.wedatasphere.dss.framework.appconn.dao,\
com.webank.wedatasphere.dss.workflow.dao,\
com.webank.wedatasphere.linkis.metadata.dao,\
com.webank.wedatasphere.exchangis.job.server.mapper,\
com.webank.wedatasphere.exchangis.job.server.dao,\
Expand Down
217 changes: 26 additions & 191 deletions db/1.1.1/exchangis_ddl.sql
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,30 @@
-- exchangis_job_ds_bind definition
DROP TABLE IF EXISTS `exchangis_job_ds_bind`;
CREATE TABLE `exchangis_job_ds_bind` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`job_id` bigint(20) NOT NULL,
`task_index` int(11) NOT NULL,
`source_ds_id` bigint(20) NOT NULL,
`sink_ds_id` bigint(20) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=59575 DEFAULT CHARSET=utf8 COLLATE=utf8_bin;


-- exchangis_job_entity definition
DROP TABLE IF EXISTS `exchangis_job_entity`;
CREATE TABLE `exchangis_job_entity` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(100) NOT NULL,
`create_time` datetime DEFAULT NULL,
`last_update_time` datetime(3) DEFAULT NULL,
`engine_type` varchar(45) DEFAULT '',
`job_labels` varchar(255) DEFAULT NULL,
`create_user` varchar(100) DEFAULT NULL,
`job_content` mediumtext,
`execute_user` varchar(100) DEFAULT '',
`job_params` text,
`job_desc` varchar(255) DEFAULT NULL,
`job_type` varchar(50) DEFAULT NULL,
`project_id` bigint(13) DEFAULT NULL,
`source` text,
`modify_user` varchar(50) DEFAULT NULL COMMENT '修改用户',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=5793 DEFAULT CHARSET=utf8;
-- exchangis_job_func definition
DROP TABLE IF EXISTS `exchangis_job_func`;
CREATE TABLE `exchangis_job_func` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`func_type` varchar(50) NOT NULL,
`func_name` varchar(100) NOT NULL,
`tab_name` varchar(50) NOT NULL COMMENT 'Tab',
`name_dispaly` varchar(100) DEFAULT NULL,
`param_num` int(11) DEFAULT '0',
`ref_name` varchar(100) DEFAULT NULL,
`description` varchar(200) DEFAULT NULL,
`modify_time` datetime DEFAULT NULL,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`)
) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8;

-- exchangis_job_func_params definition
DROP TABLE IF EXISTS `exchangis_job_func_params`;
CREATE TABLE IF NOT EXISTS `exchangis_job_func_params`(
`func_id` INT(11) NOT NULL,
`param_name` VARCHAR(100) NOT NULL,
`order` INT(11) DEFAULT 0,
`name_display` VARCHAR(100),
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(`func_id`, `param_name`)
)Engine=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_job_param_config definition
DROP TABLE IF EXISTS `exchangis_job_param_config`;
Expand Down Expand Up @@ -63,150 +57,6 @@ CREATE TABLE `exchangis_job_param_config` (
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=32 DEFAULT CHARSET=utf8;

-- exchangis_project_info definition
DROP TABLE IF EXISTS `exchangis_project_info`;
-- udes_gzpc_pub_sit_01.exchangis_project_info definition
CREATE TABLE `exchangis_project_info` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(64) NOT NULL,
`description` varchar(255) DEFAULT NULL,
`create_time` datetime DEFAULT DEFAULT CURRENT_TIMESTAMP,
`last_update_time` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`create_user` varchar(64) DEFAULT NULL,
`last_update_user` varchar(64) DEFAULT NULL,
`project_labels` varchar(255) DEFAULT NULL,
`domain` varchar(32) DEFAULT NULL,
`exec_users` varchar(255) DEFAULT '',
`view_users` varchar(255) DEFAULT '',
`edit_users` varchar(255) DEFAULT '',
`source` text,
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=1497870871035974171 DEFAULT CHARSET=utf8;

-- exchangis_project_user definition
DROP TABLE IF EXISTS `exchangis_project_user`;
CREATE TABLE `exchangis_project_user` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`project_id` bigint(20) NOT NULL,
`priv_user` varchar(32) COLLATE utf8_bin DEFAULT NULL,
`priv` int(20) DEFAULT NULL,
`last_update_time` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `exchangis_project_user_un` (`project_id`)
) ENGINE=InnoDB AUTO_INCREMENT=844 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=COMPACT;

-- exchangis_launchable_task definition
DROP TABLE IF EXISTS `exchangis_launchable_task`;
CREATE TABLE `exchangis_launchable_task` (
`id` bigint(13) NOT NULL,
`name` varchar(100) NOT NULL,
`job_execution_id` varchar(64) DEFAULT NULL,
`create_time` datetime DEFAULT NULL,
`last_update_time` datetime(3) DEFAULT NULL,
`engine_type` varchar(45) DEFAULT '',
`execute_user` varchar(50) DEFAULT '',
`linkis_job_name` varchar(100) NOT NULL,
`linkis_job_content` text NOT NULL,
`linkis_params` text DEFAULT NULL,
`linkis_source` varchar(64) DEFAULT NULL,
`labels` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_launched_job_entity definition
DROP TABLE IF EXISTS `exchangis_launched_job_entity`;
CREATE TABLE `exchangis_launched_job_entity` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`name` varchar(100) NOT NULL,
`create_time` datetime DEFAULT NULL,
`last_update_time` datetime(3) DEFAULT NULL,
`job_id` bigint(20) DEFAULT NULL,
`launchable_task_num` int(20) DEFAULT '0',
`engine_type` varchar(100) DEFAULT NULL,
`execute_user` varchar(100) DEFAULT NULL,
`job_name` varchar(100) DEFAULT NULL,
`status` varchar(100) DEFAULT NULL,
`progress` varchar(100) DEFAULT NULL,
`error_code` varchar(64) DEFAULT NULL,
`error_msg` varchar(255) DEFAULT NULL,
`retry_num` bigint(10) DEFAULT NULL,
`job_execution_id` varchar(255) DEFAULT NULL,
`log_path` varchar(255) DEFAULT NULL,
`create_user` varchar(100) DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `job_execution_id_UNIQUE` (`job_execution_id`)
) ENGINE=InnoDB AUTO_INCREMENT=8380 DEFAULT CHARSET=utf8;

-- exchangis_launched_task_entity definition
DROP TABLE IF EXISTS `exchangis_launched_task_entity`;
CREATE TABLE `exchangis_launched_task_entity` (
`id` bigint(20) NOT NULL,
`name` varchar(100) NOT NULL,
`create_time` datetime DEFAULT NULL,
`last_update_time` datetime(3) DEFAULT NULL,
`job_id` bigint(20) DEFAULT NULL,
`engine_type` varchar(100) DEFAULT NULL,
`execute_user` varchar(100) DEFAULT NULL,
`job_name` varchar(100) DEFAULT NULL,
`progress` varchar(64) DEFAULT NULL,
`error_code` varchar(64) DEFAULT NULL,
`error_msg` varchar(255) DEFAULT NULL,
`retry_num` bigint(10) DEFAULT NULL,
`task_id` varchar(64) DEFAULT NULL,
`linkis_job_id` varchar(200) DEFAULT NULL,
`linkis_job_info` varchar(1000) DEFAULT NULL,
`job_execution_id` varchar(100) DEFAULT NULL,
`launch_time` datetime DEFAULT NULL,
`running_time` datetime DEFAULT NULL,
`metrics` text,
`status` varchar(64) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_job_func definition
DROP TABLE IF EXISTS `exchangis_job_func`;
CREATE TABLE `exchangis_job_func` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`func_type` varchar(50) NOT NULL,
`func_name` varchar(100) NOT NULL,
`tab_name` varchar(50) NOT NULL COMMENT 'Tab',
`name_dispaly` varchar(100) DEFAULT NULL,
`param_num` int(11) DEFAULT '0',
`ref_name` varchar(100) DEFAULT NULL,
`description` varchar(200) DEFAULT NULL,
`modify_time` datetime DEFAULT NULL,
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `job_func_tab_name_idx` (`tab_name`,`func_name`)
) ENGINE=InnoDB AUTO_INCREMENT=12 DEFAULT CHARSET=utf8;

-- exchangis_job_func_params definition
DROP TABLE IF EXISTS `exchangis_job_func_params`;
CREATE TABLE IF NOT EXISTS `exchangis_job_func_params`(
`func_id` INT(11) NOT NULL,
`param_name` VARCHAR(100) NOT NULL,
`order` INT(11) DEFAULT 0,
`name_display` VARCHAR(100),
`create_time` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY(`func_id`, `param_name`)
)Engine=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_engine_resources definition
DROP TABLE IF EXISTS `exchangis_engine_resources`
CREATE TABLE `exchangis_engine_resources` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`engine_type` varchar(50) NOT NULL,
`resource_name` varchar(100) NOT NULL,
`resource_type` varchar(50) NOT NULL COMMENT 'resource type' DEFAULT 'file',
`resource_path` varchar(255) NOT NULL,
`store_uri` varchar(500) NOT NULL,
`create_user` varchar(50) NOT NULL,
`modify_time` datetime DEFAULT NULL,
`create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`),
UNIQUE KEY `engine_res_idx` (`engine_type`,`resource_path`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_engine_settings definition
DROP TABLE IF EXISTS `exchangis_engine_settings`;
CREATE TABLE `exchangis_engine_settings` (
Expand Down Expand Up @@ -235,19 +85,4 @@ CREATE TABLE `exchangis_job_transform_rule` (
`direction` varchar(32) NOT NULL DEFAULT 'NONE',
`create_time` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

-- exchangis_job_transform_processor
DROP TABLE IF EXISTS `exchangis_job_transform_processor`;
CREATE TABLE `exchangis_job_transform_processor` (
`id` bigint(20) NOT NULL AUTO_INCREMENT,
`job_id` bigint(20) NOT NULL,
`code_content` text DEFAULT NULL,
`code_language` varchar(32) NOT NULL DEFAULT 'java',
`code_bml_resourceId` varchar(255) COMMENT 'BML resource id',
`code_bml_version` varchar(255) COMMENT 'BML version',
`creator` varchar(50) NOT NULL COMMENT 'Owner of processor',
`create_time` datetime DEFAULT CURRENT_TIMESTAMP,
`update_time` datetime DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
40 changes: 19 additions & 21 deletions db/1.1.1/exchangis_dml.sql
View file
Original file line number Diff line number Diff line change
@@ -1,18 +1,16 @@
-- job_func records
INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES
('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL,'2020-04-21 17:35:06.000')
,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000')
,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL,'2020-04-21 17:35:06.000')
;
INSERT INTO exchangis_job_func (func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time,create_time) VALUES
('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL,'2020-04-21 17:35:06.000')
INSERT INTO `exchangis_job_func`(func_type,func_name,tab_name,name_dispaly,param_num,ref_name,description,modify_time) VALUES
('TRANSFORM','dx_substr','DATAX',NULL,2,NULL,NULL,NULL)
,('TRANSFORM','dx_pad','DATAX',NULL,3,NULL,NULL,NULL)
,('TRANSFORM','dx_replace','DATAX',NULL,3,NULL,NULL,NULL)
,('VERIFY','like','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','not like','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','>','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','<','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','=','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','!=','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('VERIFY','>=','DATAX',NULL,1,'dx_filter',NULL,NULL)
,('TRANSFORM','dx_precision','DATAX',NULL,1,NULL,NULL,NULL)
;

-- job_func_params records
Expand Down Expand Up @@ -61,7 +59,6 @@ INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_directio
('nullFormat','空值字符','DATAX-SINK','HIVE','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49)
,('nullFormat','空值字符','DATAX-SINK','ELASTICSEARCH','INPUT','nullFormat','空值字符','',0,'VARCHAR','','','REGEX','^[sS]{0,50}$','空值字符输入错误',0,0,'',1,'',2,'',1,49)
;

INSERT INTO `exchangis_job_param_config` (config_key,config_name,config_direction,`type`,ui_type,ui_field,ui_label,unit,required,value_type,value_range,default_value,validate_type,validate_range,validate_msg,is_hidden,is_advanced,source,`level`,treename,sort,description,status,ref_id) VALUES
('where','WHERE条件','SOURCE','ORACLE','INPUT','where','WHERE条件',NULL,0,'VARCHAR',NULL,NULL,'REGEX','^[\\s\\S]{0,500}$','WHERE条件输入过长',0,0,NULL,1,'',2,NULL,1,NULL)
,('writeMode','写入方式','DATAX-SINK','ORACLE','OPTION','writeMode','写入方式',NULL,1,'OPTION','["INSERT","UPDATE"]','INSERT',NULL,NULL,'写入方式输入错误',0,0,NULL,1,NULL,1,NULL,1,NULL)
Expand All @@ -72,10 +69,11 @@ INSERT INTO `exchangis_engine_settings` (id, engine_name, engine_desc, engine_se
(1, 'datax', 'datax sync engine', '{}', 'mysql->hive,hive->mysql,mysql->oracle,oracle->mysql,oracle->hive,hive->oracle,mongodb->hive,hive->mongodb,mysql->elasticsearch,oracle->elasticsearch,mongodb->elasticsearch,mysql->mongodb,mongodb->mysql,oracle->mongodb,mongodb->oracle', 'com.webank.wedatasphere.exchangis.engine.resource.loader.datax.DataxEngineResourceLoader', NULL, NULL, '2022-08-09 18:20:51.0'),
(2, 'sqoop', 'hadoop tool', '{}', 'mysql->hive,hive->mysql', '', NULL, NULL, '2022-08-09 18:20:51.0');

INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction,create_time) VALUES
('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK','2022-09-18 23:36:42.000')
,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK','2022-09-20 16:27:18.000')
,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK','2022-09-20 21:46:05.000')
,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK','2022-09-20 21:55:05.000')
,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE','2022-09-21 21:55:05.000')
-- exchangis_job_transform_rule records
INSERT INTO `exchangis_job_transform_rule` (rule_name,rule_type,rule_source,data_source_type,engine_type,direction) VALUES
('es_with_post_processor','DEF','{"types": ["MAPPING", "PROCESSOR"]}','ELASTICSEARCH',NULL,'SINK')
,('es_fields_not_editable','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false}','ELASTICSEARCH',NULL,'SINK')
,('hive_sink_not_access','MAPPING','{"fieldEditEnable": false, "fieldDeleteEnable": false, "fieldAddEnable": false}','HIVE',NULL,'SINK')
,('mongo_field_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH"}','MONGODB',NULL,'SINK')
,('mysql_field_source_match','MAPPING','{"fieldMatchStrategyName": "CAMEL_CASE_MATCH","fieldEditEnable": true, "fieldDeleteEnable": true, "fieldAddEnable": false}','MYSQL',NULL,'SOURCE')
;
Loading

0 comments on commit afdb9de

Please sign in to comment.