- mysql源表建表语句
CREATE TABLE `app_apm_app` (
`id` bigint NOT NULL AUTO_INCREMENT,
`changed` datetime(6) DEFAULT NULL,
`changerId` bigint DEFAULT NULL,
`created` datetime(6) NOT NULL,
`creatorId` bigint NOT NULL,
`description` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
`guid` varchar(36) COLLATE utf8mb4_unicode_ci NOT NULL,
`name` varchar(15) COLLATE utf8mb4_unicode_ci NOT NULL,
`runOs` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL,
`version` varchar(12) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
PRIMARY KEY (`id`),
UNIQUE KEY `UK_c7mqguh0c8di6x9f63u83weyp` (`guid`),
UNIQUE KEY `UK_q8og0jml1k3pmmkwr9ckv4eny` (`name`)
) ENGINE=InnoDB AUTO_INCREMENT=14 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci; - Hive ods层目标测试表建表语句
drop table if exists ods_app_apm_app_test;
CREATE TABLE `ods_app_apm_app_test`(
`id` bigint ,
`changed` string ,
`changerId` bigint ,
`created` string ,
`creatorId` bigint,
`description` string,
`guid` string,
`name` string,
`runOs` string,
`version` string
)
row format delimited fields terminated by ‘\u0001’; - 在hadoopS01、hadoopS02 、hadoopS03、hadoopS04、hadoopS05的 /mnt/sdg/software/datax/job目录下新增datax的json文件
- {
- “job”: {
- “content”: [
- {
- “reader”: {
- “name”: “mysqlreader”,
- “parameter”: {
- “column”: [
- “id”,
- “changed”,
- “changerId”,
- “created”,
- “creatorId”,
- “description”,
- “guid”,
- “name”,
- “runOs”,
- “version”
- ],
- “connection”: [
- {
- “jdbcUrl”: [
- “jdbc:mysql://xxx:3306/FDMAP”
- ],
- “table”: [
- “app_apm_app”
- ]
- “jdbcUrl”: [
- }
- {
- ],
- “username”: “shuke_etl”,
- “password”: “FUdJkwkL53”
- “column”: [
- }
- },
- “writer”: {
- “name”: “hdfswriter”,
- “parameter”: {
- “column”: [
- {
- “name”: “id”,
- “type”: “BIGINT”
- },
- {
- “name”: “changed”,
- “type”: “STRING”
- },
- {
- “name”: “changerId”,
- “type”: “BIGINT”
- },
- {
- “name”: “created”,
- “type”: “STRING”
- },
- {
- “name”: “creatorId”,
- “type”: “BIGINT”
- },
- {
- “name”: “description”,
- “type”: “STRING”
- },
- {
- “name”: “guid”,
- “type”: “STRING”
- },
- {
- “name”: “name”,
- “type”: “STRING”
- },
- {
- “name”: “runOs”,
- “type”: “STRING”
- },
- {
- “name”: “version”,
- “type”: “STRING”
- }
- {
- ],
- “defaultFS”: “hdfs://nameservice1”,
- “column”: [
- “reader”: {
- “hadoopConfig”:{
- “dfs.nameservices”: “nameservice1”,
- “dfs.ha.namenodes.nameservice”: “namenode69,namenode80”,
- “dfs.namenode.rpc-address.nameservice.namenode69”: “hadoopm01:8020”,
- “dfs.namenode.rpc-address.nameservice.namenode80”: “hadoopm02:8020”,
- “dfs.client.failover.proxy.provider.nameservice”: “org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider”,
- “dfs.client.failover.proxy.provider.nameservice.configured.providers”: “namenode69,namenode80”,
- “dfs.client.failover.proxy.provider.namenode69”: “org.apache.hadoop.hdfs.server.namenode.ha.KerberosFailoverProxyProvider”,
- “dfs.client.failover.proxy.provider.namenode80”: “org.apache.hadoop.hdfs.server.namenode.ha.KerberosFailoverProxyProvider”
-
- },
-
-
-
- “path”: “/user/hive/warehouse/test.db/ods_app_apm_app_test/”,
-
-
-
- },
-
- “fileType”: “text”,
-
-
-
-
-
- “fileName”: “ods_app_apm_app_test.txt”,
-
-
-
-
-
- “encoding”: “UTF-8”,
- “fieldDelimiter”: “\u0001”,
-
- “haveKerberos”: “true”,
-
- “kerberosKeytabFilePath”: “/opt/kerberos_keytab/dolphinscheduler.keytab”,
- “kerberosPrincipal”: “hdfs/hdfs@HADOOP.COM”,
-
-
-
-
-
- “writeMode”: “append”
- }
-
- }
-
- }
-
- ],
- “setting”: {
- “speed”: {
- “channel”: “1”
- }
- “speed”: {
- }
-
- }
- }
-
- {
- “content”: [
- “job”: {
- 进入到hdfsreader的文件夹中，运行datax任务
cd /mnt/sdg/software/datax/plugin/reader/hdfsreader
python /mnt/sdg/software/datax/bin/datax.py /mnt/sdg/software/datax/job/mysql2hdfs_test.json
作者 admin
张宴银，大数据开发工程师